From 9011fd40e8bfbaf36f940c256f5e028c0582c1de Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 22 Jan 2024 14:54:20 +0100 Subject: [PATCH 1/6] Initial commit --- src/commons/product_image_selection.rs | 5 ++++ src/helm.rs | 39 ++++++++++++++++++++++++++ src/lib.rs | 1 + 3 files changed, 45 insertions(+) create mode 100644 src/helm.rs diff --git a/src/commons/product_image_selection.rs b/src/commons/product_image_selection.rs index 17eb58c8d..a999febfa 100644 --- a/src/commons/product_image_selection.rs +++ b/src/commons/product_image_selection.rs @@ -85,6 +85,11 @@ pub struct ResolvedProductImage { #[derive(AsRefStr)] /// We default to `Always`, as we use floating tags for our release lines. /// This means the tag 23.4 starts of pointing to the same image 23.4.0 does, but switches to 23.4.1 after the releases of 23.4.1. +/// +/// ### See +/// +/// - +/// - pub enum PullPolicy { IfNotPresent, #[default] diff --git a/src/helm.rs b/src/helm.rs new file mode 100644 index 000000000..59f22fe11 --- /dev/null +++ b/src/helm.rs @@ -0,0 +1,39 @@ +use std::collections::BTreeMap; + +use k8s_openapi::api::core::v1::LocalObjectReference; +use serde::{Deserialize, Serialize}; + +use crate::commons::product_image_selection::PullPolicy; + +/// A dynamic representation of a Helm `values.yaml` file. +/// +/// This will work with any operator `values.yaml` file. It basically only +/// contains common fields which exist in all value files we use for our +/// operators. +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DynamicValues { + pub image: ImageValues, + pub name_override: String, + pub full_name_override: String, + pub service_account: ServiceAccountValues, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct ImageValues { + /// Specify from which repository this image should get pulled from + pub repository: String, + + /// Specify the pull policy of this image. See more at + /// + pub pull_policy: PullPolicy, + pub pull_secrets: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ServiceAccountValues { + pub create: bool, + pub annotations: BTreeMap, + pub name: String, +} diff --git a/src/lib.rs b/src/lib.rs index 4d3e5116c..86eb247b4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,6 +7,7 @@ pub mod config; pub mod cpu; pub mod crd; pub mod error; +pub mod helm; pub mod iter; pub mod kvp; pub mod logging; From 3ef497c12e3addecbed97412ad6d741ea1bd44ea Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 24 Jan 2024 10:43:53 +0100 Subject: [PATCH 2/6] Finish common set of keys --- src/helm.rs | 39 --------------------- src/helm/mod.rs | 91 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 39 deletions(-) delete mode 100644 src/helm.rs create mode 100644 src/helm/mod.rs diff --git a/src/helm.rs b/src/helm.rs deleted file mode 100644 index 59f22fe11..000000000 --- a/src/helm.rs +++ /dev/null @@ -1,39 +0,0 @@ -use std::collections::BTreeMap; - -use k8s_openapi::api::core::v1::LocalObjectReference; -use serde::{Deserialize, Serialize}; - -use crate::commons::product_image_selection::PullPolicy; - -/// A dynamic representation of a Helm `values.yaml` file. -/// -/// This will work with any operator `values.yaml` file. It basically only -/// contains common fields which exist in all value files we use for our -/// operators. -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DynamicValues { - pub image: ImageValues, - pub name_override: String, - pub full_name_override: String, - pub service_account: ServiceAccountValues, -} - -#[derive(Debug, Default, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", default)] -pub struct ImageValues { - /// Specify from which repository this image should get pulled from - pub repository: String, - - /// Specify the pull policy of this image. See more at - /// - pub pull_policy: PullPolicy, - pub pull_secrets: Vec, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct ServiceAccountValues { - pub create: bool, - pub annotations: BTreeMap, - pub name: String, -} diff --git a/src/helm/mod.rs b/src/helm/mod.rs new file mode 100644 index 000000000..5af1a2221 --- /dev/null +++ b/src/helm/mod.rs @@ -0,0 +1,91 @@ +use std::collections::BTreeMap; + +use k8s_openapi::api::core::v1::LocalObjectReference; +use serde::{Deserialize, Serialize}; + +use crate::{ + commons::product_image_selection::PullPolicy, cpu::CpuQuantity, memory::MemoryQuantity, +}; + +static EMPTY_MAP: BTreeMap = BTreeMap::new(); + +/// A dynamic representation of a Helm `values.yaml` file. +/// +/// This will work with any operator `values.yaml` file. It basically only +/// contains common fields which exist in all value files we use for our +/// operators. +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DynamicValues { + pub image: ImageValues, + pub name_override: String, + pub full_name_override: String, + pub service_account: ServiceAccountValues, + pub resources: ResourceValues, + + // TODO(Techassi): Here we could use direct Serialize and Deserialize support + pub labels: Option>, + + /// All other keys and values. + #[serde(flatten)] + pub data: serde_yaml::Value, +} + +impl DynamicValues { + pub fn labels(&self) -> &BTreeMap { + self.labels.as_ref().unwrap_or(&EMPTY_MAP) + } + + pub fn labels_mut(&mut self) -> &mut BTreeMap { + self.labels.get_or_insert_with(BTreeMap::new) + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ImageValues { + /// Specify from which repository this image should get pulled from. + pub repository: String, + + /// Specify the pull policy of this image. + /// + /// See more at + /// + pub pull_policy: PullPolicy, + + /// Specify one or more pull secrets. + /// + /// See more at + /// + pub pull_secrets: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ServiceAccountValues { + /// Specify whether a service account should be created. + pub create: bool, + + /// Specify which annotations to add to the service account. + pub annotations: BTreeMap, + + /// Specify the name of the service account. + /// + /// If this is not set and `create` is true, a name is generated using the + /// fullname template. + pub name: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResourceValues { + limits: ComputeResourceValues, + requests: ComputeResourceValues, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ComputeResourceValues { + cpu: CpuQuantity, + memory: MemoryQuantity, +} From 0d6ba69345cefa227adc2fe3fa43f5636e267f06 Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 24 Jan 2024 10:48:36 +0100 Subject: [PATCH 3/6] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95ae7c97a..fd9d63728 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,9 @@ All notable changes to this project will be documented in this file. ### Added - Add Serde `Deserialize` and `Serialize` support for `CpuQuantity` and `MemoryQuantity` ([#724]). +- Add `DynamicValues` struct to work with operator `values.yaml` files during runtime ([#723]). +[#723]: https://github.com/stackabletech/operator-rs/pull/723 [#724]: https://github.com/stackabletech/operator-rs/pull/724 ## [0.62.0] - 2024-01-19 From 363ae711924ccd455b630280f6187debe62f0002 Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 24 Jan 2024 17:02:50 +0100 Subject: [PATCH 4/6] Add unit tests for serialization / deserialization of DynamicValues I will convert the input and output to fixture files in the next commit. --- src/helm/mod.rs | 87 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/src/helm/mod.rs b/src/helm/mod.rs index 5af1a2221..ffa7b9a69 100644 --- a/src/helm/mod.rs +++ b/src/helm/mod.rs @@ -19,7 +19,7 @@ static EMPTY_MAP: BTreeMap = BTreeMap::new(); pub struct DynamicValues { pub image: ImageValues, pub name_override: String, - pub full_name_override: String, + pub fullname_override: String, pub service_account: ServiceAccountValues, pub resources: ResourceValues, @@ -89,3 +89,88 @@ pub struct ComputeResourceValues { cpu: CpuQuantity, memory: MemoryQuantity, } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_deserialize() { + // Taken from https://github.com/stackabletech/zookeeper-operator/blob/main/deploy/helm/zookeeper-operator/values.yaml + let input = "--- +image: + repository: docker.stackable.tech/stackable/zookeeper-operator + pullPolicy: IfNotPresent + pullSecrets: [] +nameOverride: '' +fullnameOverride: '' +serviceAccount: + create: true + annotations: {} + name: '' +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi +labels: + stackable.tech/vendor: Stackable + stackable.tech/managed-by: stackablectl"; + + let values: DynamicValues = serde_yaml::from_str(input).unwrap(); + assert_eq!(values.labels().len(), 2); + } + + #[test] + fn test_serialize() { + // Taken from https://github.com/stackabletech/zookeeper-operator/blob/main/deploy/helm/zookeeper-operator/values.yaml + let input = "--- +image: + repository: docker.stackable.tech/stackable/zookeeper-operator + pullPolicy: IfNotPresent + pullSecrets: [] +nameOverride: '' +fullnameOverride: '' +serviceAccount: + create: true + annotations: {} + name: '' +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi"; + + let mut values: DynamicValues = serde_yaml::from_str(input).unwrap(); + values + .labels_mut() + .insert("stackable.tech/vendor".into(), "Stackable".into()); + + let output = "image: + repository: docker.stackable.tech/stackable/zookeeper-operator + pullPolicy: IfNotPresent + pullSecrets: [] +nameOverride: '' +fullnameOverride: '' +serviceAccount: + create: true + annotations: {} + name: '' +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi +labels: + stackable.tech/vendor: Stackable +"; + + assert_eq!(serde_yaml::to_string(&values).unwrap(), output); + } +} From 10642a4923ab8f19818fa74b1c3966fad30ab432 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 25 Jan 2024 09:44:38 +0100 Subject: [PATCH 5/6] Move test inputs and output into fixture files --- fixtures/helm/input-additional.yaml | 27 +++++++++ fixtures/helm/input-required.yaml | 22 ++++++++ fixtures/helm/output.yaml | 21 +++++++ src/helm/mod.rs | 86 +++++------------------------ 4 files changed, 85 insertions(+), 71 deletions(-) create mode 100644 fixtures/helm/input-additional.yaml create mode 100644 fixtures/helm/input-required.yaml create mode 100644 fixtures/helm/output.yaml diff --git a/fixtures/helm/input-additional.yaml b/fixtures/helm/input-additional.yaml new file mode 100644 index 000000000..72eb1fb31 --- /dev/null +++ b/fixtures/helm/input-additional.yaml @@ -0,0 +1,27 @@ +# Partly taken from https://github.com/stackabletech/zookeeper-operator/blob/main/deploy/helm/zookeeper-operator/values.yaml +--- +image: + repository: docker.stackable.tech/stackable/zookeeper-operator + pullPolicy: IfNotPresent + pullSecrets: [] +nameOverride: '' +fullnameOverride: '' +serviceAccount: + create: true + annotations: {} + name: '' +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi +labels: + stackable.tech/vendor: Stackable + stackable.tech/managed-by: stackablectl +podAnnotations: {} +podSecurityContext: {} +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/fixtures/helm/input-required.yaml b/fixtures/helm/input-required.yaml new file mode 100644 index 000000000..d8bd90f7d --- /dev/null +++ b/fixtures/helm/input-required.yaml @@ -0,0 +1,22 @@ +# Partly taken from https://github.com/stackabletech/zookeeper-operator/blob/main/deploy/helm/zookeeper-operator/values.yaml +--- +image: + repository: docker.stackable.tech/stackable/zookeeper-operator + pullPolicy: IfNotPresent + pullSecrets: [] +nameOverride: '' +fullnameOverride: '' +serviceAccount: + create: true + annotations: {} + name: '' +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi +labels: + stackable.tech/vendor: Stackable + stackable.tech/managed-by: stackablectl diff --git a/fixtures/helm/output.yaml b/fixtures/helm/output.yaml new file mode 100644 index 000000000..3b290d651 --- /dev/null +++ b/fixtures/helm/output.yaml @@ -0,0 +1,21 @@ +image: + repository: docker.stackable.tech/stackable/zookeeper-operator + pullPolicy: IfNotPresent + pullSecrets: [] +nameOverride: '' +fullnameOverride: '' +serviceAccount: + create: true + annotations: {} + name: '' +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi +labels: + stackable.tech/demo: logging + stackable.tech/managed-by: stackablectl + stackable.tech/vendor: Stackable diff --git a/src/helm/mod.rs b/src/helm/mod.rs index ffa7b9a69..fae456828 100644 --- a/src/helm/mod.rs +++ b/src/helm/mod.rs @@ -92,85 +92,29 @@ pub struct ComputeResourceValues { #[cfg(test)] mod test { + use std::path::PathBuf; + + use rstest::rstest; + use super::*; - #[test] - fn test_deserialize() { - // Taken from https://github.com/stackabletech/zookeeper-operator/blob/main/deploy/helm/zookeeper-operator/values.yaml - let input = "--- -image: - repository: docker.stackable.tech/stackable/zookeeper-operator - pullPolicy: IfNotPresent - pullSecrets: [] -nameOverride: '' -fullnameOverride: '' -serviceAccount: - create: true - annotations: {} - name: '' -resources: - limits: - cpu: 100m - memory: 128Mi - requests: - cpu: 100m - memory: 128Mi -labels: - stackable.tech/vendor: Stackable - stackable.tech/managed-by: stackablectl"; - - let values: DynamicValues = serde_yaml::from_str(input).unwrap(); + #[rstest] + fn test_deserialize(#[files("fixtures/helm/input-*.yaml")] path: PathBuf) { + let contents = std::fs::read_to_string(path).unwrap(); + let values: DynamicValues = serde_yaml::from_str(&contents).unwrap(); assert_eq!(values.labels().len(), 2); } - #[test] - fn test_serialize() { - // Taken from https://github.com/stackabletech/zookeeper-operator/blob/main/deploy/helm/zookeeper-operator/values.yaml - let input = "--- -image: - repository: docker.stackable.tech/stackable/zookeeper-operator - pullPolicy: IfNotPresent - pullSecrets: [] -nameOverride: '' -fullnameOverride: '' -serviceAccount: - create: true - annotations: {} - name: '' -resources: - limits: - cpu: 100m - memory: 128Mi - requests: - cpu: 100m - memory: 128Mi"; - - let mut values: DynamicValues = serde_yaml::from_str(input).unwrap(); + #[rstest] + fn test_serialize(#[files("fixtures/helm/input-required.yaml")] input: PathBuf) { + let contents = std::fs::read_to_string(input).unwrap(); + let mut values: DynamicValues = serde_yaml::from_str(&contents).unwrap(); + values .labels_mut() - .insert("stackable.tech/vendor".into(), "Stackable".into()); - - let output = "image: - repository: docker.stackable.tech/stackable/zookeeper-operator - pullPolicy: IfNotPresent - pullSecrets: [] -nameOverride: '' -fullnameOverride: '' -serviceAccount: - create: true - annotations: {} - name: '' -resources: - limits: - cpu: 100m - memory: 128Mi - requests: - cpu: 100m - memory: 128Mi -labels: - stackable.tech/vendor: Stackable -"; + .insert("stackable.tech/demo".into(), "logging".into()); + let output = std::fs::read_to_string("fixtures/helm/output.yaml").unwrap(); assert_eq!(serde_yaml::to_string(&values).unwrap(), output); } } From acb68a3a17f479f19a6a4397d10f5e0d70df4392 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 25 Jan 2024 09:50:13 +0100 Subject: [PATCH 6/6] Fix yamllint error --- fixtures/helm/output.yaml | 1 + src/helm/mod.rs | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/fixtures/helm/output.yaml b/fixtures/helm/output.yaml index 3b290d651..6ac527fe1 100644 --- a/fixtures/helm/output.yaml +++ b/fixtures/helm/output.yaml @@ -1,3 +1,4 @@ +--- image: repository: docker.stackable.tech/stackable/zookeeper-operator pullPolicy: IfNotPresent diff --git a/src/helm/mod.rs b/src/helm/mod.rs index fae456828..5e5db5d19 100644 --- a/src/helm/mod.rs +++ b/src/helm/mod.rs @@ -96,6 +96,8 @@ mod test { use rstest::rstest; + use crate::yaml::serialize_to_explicit_document; + use super::*; #[rstest] @@ -114,7 +116,11 @@ mod test { .labels_mut() .insert("stackable.tech/demo".into(), "logging".into()); - let output = std::fs::read_to_string("fixtures/helm/output.yaml").unwrap(); - assert_eq!(serde_yaml::to_string(&values).unwrap(), output); + let expected = std::fs::read_to_string("fixtures/helm/output.yaml").unwrap(); + + let mut output = Vec::new(); + serialize_to_explicit_document(&mut output, &values).unwrap(); + + assert_eq!(std::str::from_utf8(&output).unwrap(), expected); } }