From 443558473b3b81bda6d60b549b28d9ee7ffcc306 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 14 Aug 2024 17:11:26 +0200 Subject: [PATCH 01/12] fixed envOverrides and extended test --- rust/crd/src/lib.rs | 30 +++++++++++++++++++ rust/crd/src/s3logdir.rs | 3 +- .../src/spark_k8s_controller.rs | 6 ++-- .../pod_overrides/10-deploy-spark-app.yaml.j2 | 9 ++++++ .../kuttl/pod_overrides/11-assert.yaml | 15 ++++++++++ 5 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 tests/templates/kuttl/pod_overrides/11-assert.yaml diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 6fee239c..30ff03e1 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -727,6 +727,36 @@ impl SparkApplication { } } + pub fn merged_env(&self, role: SparkApplicationRole, env: &[EnvVar]) -> Vec { + // use a BTree internally to enable replacement of existing keys + let mut env_vars: BTreeMap = BTreeMap::new(); + + for e in env { + env_vars.insert(e.clone().name, e.to_owned()); + } + + if let Some(env_map) = match role { + SparkApplicationRole::Submit => self.spec.job.clone().map(|j| j.env_overrides), + SparkApplicationRole::Driver => self.spec.driver.clone().map(|d| d.env_overrides), + SparkApplicationRole::Executor => { + self.spec.executor.clone().map(|r| r.config.env_overrides) + } + } { + for (k, v) in env_map { + env_vars.insert( + k.clone(), + EnvVar { + name: k, + value: Some(v), + value_from: None, + }, + ); + } + } + + env_vars.into_values().collect() + } + pub fn validated_role_config( &self, resolved_product_image: &ResolvedProductImage, diff --git a/rust/crd/src/s3logdir.rs b/rust/crd/src/s3logdir.rs index f3ff8ecb..15944993 100644 --- a/rust/crd/src/s3logdir.rs +++ b/rust/crd/src/s3logdir.rs @@ -117,7 +117,8 @@ impl S3LogDir { /// * spark.hadoop.fs.s3a.aws.credentials.provider /// * spark.hadoop.fs.s3a.access.key /// * spark.hadoop.fs.s3a.secret.key - /// instead, the environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are set + /// + /// Instead, the environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are set /// on the container start command. pub fn history_server_spark_config(&self) -> BTreeMap { let mut result = BTreeMap::new(); diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index ab394117..e9a794b5 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -479,9 +479,10 @@ fn pod_template( ) -> Result { let container_name = SparkContainer::Spark.to_string(); let mut cb = ContainerBuilder::new(&container_name).context(IllegalContainerNameSnafu)?; + let merged_env = spark_application.merged_env(role.clone(), env); cb.add_volume_mounts(config.volume_mounts(spark_application, s3conn, s3logdir)) - .add_env_vars(env.to_vec()) + .add_env_vars(merged_env) .resources(config.resources.clone().into()) .image_from_product_image(spark_image); @@ -716,13 +717,14 @@ fn spark_job( .context(IllegalContainerNameSnafu)?; let args = [job_commands.join(" ")]; + let merged_env = spark_application.merged_env(SparkApplicationRole::Submit, env); cb.image_from_product_image(spark_image) .command(vec!["/bin/bash".to_string(), "-c".to_string()]) .args(vec![args.join(" && ")]) .resources(job_config.resources.clone().into()) .add_volume_mounts(spark_application.spark_job_volume_mounts(s3conn, s3logdir)) - .add_env_vars(env.to_vec()) + .add_env_vars(merged_env) .add_env_var( "SPARK_SUBMIT_OPTS", format!( diff --git a/tests/templates/kuttl/pod_overrides/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/pod_overrides/10-deploy-spark-app.yaml.j2 index 8858fadd..e9cc4f62 100644 --- a/tests/templates/kuttl/pod_overrides/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/pod_overrides/10-deploy-spark-app.yaml.j2 @@ -24,7 +24,14 @@ spec: prefix: eventlogs/ bucket: reference: spark-history-s3-bucket + env: + - name: TEST_SPARK_VAR_0 + value: ORIGINAL + - name: TEST_SPARK_VAR_1 + value: DONOTREPLACE job: + envOverrides: &envOverrides + TEST_SPARK_VAR_0: REPLACED podOverrides: spec: containers: @@ -37,6 +44,7 @@ spec: cpu: 1500m memory: 1024Mi driver: + envOverrides: *envOverrides podOverrides: spec: containers: @@ -50,6 +58,7 @@ spec: memory: 1024Mi executor: replicas: 1 + envOverrides: *envOverrides podOverrides: spec: containers: diff --git a/tests/templates/kuttl/pod_overrides/11-assert.yaml b/tests/templates/kuttl/pod_overrides/11-assert.yaml new file mode 100644 index 00000000..87871e41 --- /dev/null +++ b/tests/templates/kuttl/pod_overrides/11-assert.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +commands: + - script: kubectl -n $NAMESPACE get job spark-pi-s3-1 -o yaml | yq '.spec.template.spec.containers[0].env[] | select (.name == "TEST_SPARK_VAR_0").value' | grep 'REPLACED' + - script: kubectl -n $NAMESPACE get job spark-pi-s3-1 -o yaml | yq '.spec.template.spec.containers[0].env[] | select (.name == "TEST_SPARK_VAR_1").value' | grep 'DONOTREPLACE' + - script: kubectl -n $NAMESPACE get cm spark-pi-s3-1-driver-pod-template -o json | jq -r '.data."template.yaml"' | yq '.spec.containers[0].env[] | select (.name == "TEST_SPARK_VAR_0").value' | grep 'REPLACED' + - script: kubectl -n $NAMESPACE get cm spark-pi-s3-1-driver-pod-template -o json | jq -r '.data."template.yaml"' | yq '.spec.containers[0].env[] | select (.name == "TEST_SPARK_VAR_1").value' | grep 'DONOTREPLACE' + - script: kubectl -n $NAMESPACE get cm spark-pi-s3-1-executor-pod-template -o json | jq -r '.data."template.yaml"' | yq '.spec.containers[0].env[] | select (.name == "TEST_SPARK_VAR_0").value' | grep 'REPLACED' + - script: kubectl -n $NAMESPACE get cm spark-pi-s3-1-executor-pod-template -o json | jq -r '.data."template.yaml"' | yq '.spec.containers[0].env[] | select (.name == "TEST_SPARK_VAR_1").value' | grep 'DONOTREPLACE' + - script: | + POD=$(kubectl -n $NAMESPACE get pod -l app.kubernetes.io/instance=spark-pi-s3-1 -o name | head -n 1 | sed -e 's#pod/##') + kubectl -n $NAMESPACE get pod $POD -o yaml | yq '.spec.containers[0].env[] | select (.name == "TEST_SPARK_VAR_0").value' | grep 'REPLACED' + kubectl -n $NAMESPACE get pod $POD -o yaml | yq '.spec.containers[0].env[] | select (.name == "TEST_SPARK_VAR_1").value' | grep 'DONOTREPLACE' From c87f14c8b9b3346a819ff5f3ae45fc550ae5d078 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 14 Aug 2024 17:45:06 +0200 Subject: [PATCH 02/12] renamed test as it now covers env- and pod-overrides --- .../templates/kuttl/{pod_overrides => overrides}/00-assert.yaml | 0 .../kuttl/{pod_overrides => overrides}/00-patch-ns.yaml.j2 | 0 .../kuttl/{pod_overrides => overrides}/00-secrets.yaml.j2 | 0 .../{pod_overrides => overrides}/00-serviceaccount.yaml.j2 | 0 .../templates/kuttl/{pod_overrides => overrides}/03-assert.yaml | 0 .../kuttl/{pod_overrides => overrides}/03-setup-minio.yaml | 0 .../kuttl/{pod_overrides => overrides}/05-s3-connection.yaml.j2 | 0 .../templates/kuttl/{pod_overrides => overrides}/06-assert.yaml | 0 .../06-deploy-history-server.yaml.j2 | 0 .../templates/kuttl/{pod_overrides => overrides}/10-assert.yaml | 0 .../{pod_overrides => overrides}/10-deploy-spark-app.yaml.j2 | 0 .../templates/kuttl/{pod_overrides => overrides}/11-assert.yaml | 0 .../templates/kuttl/{pod_overrides => overrides}/20-assert.yaml | 0 .../kuttl/{pod_overrides => overrides}/20-test-logs.yaml | 0 tests/templates/kuttl/{pod_overrides => overrides}/certs/ca.crt | 0 .../kuttl/{pod_overrides => overrides}/certs/client.crt.pem | 0 .../kuttl/{pod_overrides => overrides}/certs/client.csr.pem | 0 .../kuttl/{pod_overrides => overrides}/certs/client.key.pem | 0 .../kuttl/{pod_overrides => overrides}/certs/generate.sh | 0 .../kuttl/{pod_overrides => overrides}/certs/root-ca.crt.pem | 0 .../kuttl/{pod_overrides => overrides}/certs/root-ca.crt.srl | 0 .../kuttl/{pod_overrides => overrides}/certs/root-ca.key.pem | 0 .../templates/kuttl/{pod_overrides => overrides}/certs/tls.crt | 0 .../templates/kuttl/{pod_overrides => overrides}/certs/tls.key | 0 .../helm-bitnami-eventlog-minio-values.yaml.j2 | 0 .../{pod_overrides => overrides}/helm-bitnami-minio-values.yaml | 0 tests/test-definition.yaml | 2 +- 27 files changed, 1 insertion(+), 1 deletion(-) rename tests/templates/kuttl/{pod_overrides => overrides}/00-assert.yaml (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/00-patch-ns.yaml.j2 (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/00-secrets.yaml.j2 (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/00-serviceaccount.yaml.j2 (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/03-assert.yaml (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/03-setup-minio.yaml (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/05-s3-connection.yaml.j2 (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/06-assert.yaml (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/06-deploy-history-server.yaml.j2 (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/10-assert.yaml (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/10-deploy-spark-app.yaml.j2 (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/11-assert.yaml (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/20-assert.yaml (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/20-test-logs.yaml (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/certs/ca.crt (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/certs/client.crt.pem (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/certs/client.csr.pem (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/certs/client.key.pem (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/certs/generate.sh (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/certs/root-ca.crt.pem (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/certs/root-ca.crt.srl (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/certs/root-ca.key.pem (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/certs/tls.crt (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/certs/tls.key (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/helm-bitnami-eventlog-minio-values.yaml.j2 (100%) rename tests/templates/kuttl/{pod_overrides => overrides}/helm-bitnami-minio-values.yaml (100%) diff --git a/tests/templates/kuttl/pod_overrides/00-assert.yaml b/tests/templates/kuttl/overrides/00-assert.yaml similarity index 100% rename from tests/templates/kuttl/pod_overrides/00-assert.yaml rename to tests/templates/kuttl/overrides/00-assert.yaml diff --git a/tests/templates/kuttl/pod_overrides/00-patch-ns.yaml.j2 b/tests/templates/kuttl/overrides/00-patch-ns.yaml.j2 similarity index 100% rename from tests/templates/kuttl/pod_overrides/00-patch-ns.yaml.j2 rename to tests/templates/kuttl/overrides/00-patch-ns.yaml.j2 diff --git a/tests/templates/kuttl/pod_overrides/00-secrets.yaml.j2 b/tests/templates/kuttl/overrides/00-secrets.yaml.j2 similarity index 100% rename from tests/templates/kuttl/pod_overrides/00-secrets.yaml.j2 rename to tests/templates/kuttl/overrides/00-secrets.yaml.j2 diff --git a/tests/templates/kuttl/pod_overrides/00-serviceaccount.yaml.j2 b/tests/templates/kuttl/overrides/00-serviceaccount.yaml.j2 similarity index 100% rename from tests/templates/kuttl/pod_overrides/00-serviceaccount.yaml.j2 rename to tests/templates/kuttl/overrides/00-serviceaccount.yaml.j2 diff --git a/tests/templates/kuttl/pod_overrides/03-assert.yaml b/tests/templates/kuttl/overrides/03-assert.yaml similarity index 100% rename from tests/templates/kuttl/pod_overrides/03-assert.yaml rename to tests/templates/kuttl/overrides/03-assert.yaml diff --git a/tests/templates/kuttl/pod_overrides/03-setup-minio.yaml b/tests/templates/kuttl/overrides/03-setup-minio.yaml similarity index 100% rename from tests/templates/kuttl/pod_overrides/03-setup-minio.yaml rename to tests/templates/kuttl/overrides/03-setup-minio.yaml diff --git a/tests/templates/kuttl/pod_overrides/05-s3-connection.yaml.j2 b/tests/templates/kuttl/overrides/05-s3-connection.yaml.j2 similarity index 100% rename from tests/templates/kuttl/pod_overrides/05-s3-connection.yaml.j2 rename to tests/templates/kuttl/overrides/05-s3-connection.yaml.j2 diff --git a/tests/templates/kuttl/pod_overrides/06-assert.yaml b/tests/templates/kuttl/overrides/06-assert.yaml similarity index 100% rename from tests/templates/kuttl/pod_overrides/06-assert.yaml rename to tests/templates/kuttl/overrides/06-assert.yaml diff --git a/tests/templates/kuttl/pod_overrides/06-deploy-history-server.yaml.j2 b/tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 similarity index 100% rename from tests/templates/kuttl/pod_overrides/06-deploy-history-server.yaml.j2 rename to tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 diff --git a/tests/templates/kuttl/pod_overrides/10-assert.yaml b/tests/templates/kuttl/overrides/10-assert.yaml similarity index 100% rename from tests/templates/kuttl/pod_overrides/10-assert.yaml rename to tests/templates/kuttl/overrides/10-assert.yaml diff --git a/tests/templates/kuttl/pod_overrides/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/overrides/10-deploy-spark-app.yaml.j2 similarity index 100% rename from tests/templates/kuttl/pod_overrides/10-deploy-spark-app.yaml.j2 rename to tests/templates/kuttl/overrides/10-deploy-spark-app.yaml.j2 diff --git a/tests/templates/kuttl/pod_overrides/11-assert.yaml b/tests/templates/kuttl/overrides/11-assert.yaml similarity index 100% rename from tests/templates/kuttl/pod_overrides/11-assert.yaml rename to tests/templates/kuttl/overrides/11-assert.yaml diff --git a/tests/templates/kuttl/pod_overrides/20-assert.yaml b/tests/templates/kuttl/overrides/20-assert.yaml similarity index 100% rename from tests/templates/kuttl/pod_overrides/20-assert.yaml rename to tests/templates/kuttl/overrides/20-assert.yaml diff --git a/tests/templates/kuttl/pod_overrides/20-test-logs.yaml b/tests/templates/kuttl/overrides/20-test-logs.yaml similarity index 100% rename from tests/templates/kuttl/pod_overrides/20-test-logs.yaml rename to tests/templates/kuttl/overrides/20-test-logs.yaml diff --git a/tests/templates/kuttl/pod_overrides/certs/ca.crt b/tests/templates/kuttl/overrides/certs/ca.crt similarity index 100% rename from tests/templates/kuttl/pod_overrides/certs/ca.crt rename to tests/templates/kuttl/overrides/certs/ca.crt diff --git a/tests/templates/kuttl/pod_overrides/certs/client.crt.pem b/tests/templates/kuttl/overrides/certs/client.crt.pem similarity index 100% rename from tests/templates/kuttl/pod_overrides/certs/client.crt.pem rename to tests/templates/kuttl/overrides/certs/client.crt.pem diff --git a/tests/templates/kuttl/pod_overrides/certs/client.csr.pem b/tests/templates/kuttl/overrides/certs/client.csr.pem similarity index 100% rename from tests/templates/kuttl/pod_overrides/certs/client.csr.pem rename to tests/templates/kuttl/overrides/certs/client.csr.pem diff --git a/tests/templates/kuttl/pod_overrides/certs/client.key.pem b/tests/templates/kuttl/overrides/certs/client.key.pem similarity index 100% rename from tests/templates/kuttl/pod_overrides/certs/client.key.pem rename to tests/templates/kuttl/overrides/certs/client.key.pem diff --git a/tests/templates/kuttl/pod_overrides/certs/generate.sh b/tests/templates/kuttl/overrides/certs/generate.sh similarity index 100% rename from tests/templates/kuttl/pod_overrides/certs/generate.sh rename to tests/templates/kuttl/overrides/certs/generate.sh diff --git a/tests/templates/kuttl/pod_overrides/certs/root-ca.crt.pem b/tests/templates/kuttl/overrides/certs/root-ca.crt.pem similarity index 100% rename from tests/templates/kuttl/pod_overrides/certs/root-ca.crt.pem rename to tests/templates/kuttl/overrides/certs/root-ca.crt.pem diff --git a/tests/templates/kuttl/pod_overrides/certs/root-ca.crt.srl b/tests/templates/kuttl/overrides/certs/root-ca.crt.srl similarity index 100% rename from tests/templates/kuttl/pod_overrides/certs/root-ca.crt.srl rename to tests/templates/kuttl/overrides/certs/root-ca.crt.srl diff --git a/tests/templates/kuttl/pod_overrides/certs/root-ca.key.pem b/tests/templates/kuttl/overrides/certs/root-ca.key.pem similarity index 100% rename from tests/templates/kuttl/pod_overrides/certs/root-ca.key.pem rename to tests/templates/kuttl/overrides/certs/root-ca.key.pem diff --git a/tests/templates/kuttl/pod_overrides/certs/tls.crt b/tests/templates/kuttl/overrides/certs/tls.crt similarity index 100% rename from tests/templates/kuttl/pod_overrides/certs/tls.crt rename to tests/templates/kuttl/overrides/certs/tls.crt diff --git a/tests/templates/kuttl/pod_overrides/certs/tls.key b/tests/templates/kuttl/overrides/certs/tls.key similarity index 100% rename from tests/templates/kuttl/pod_overrides/certs/tls.key rename to tests/templates/kuttl/overrides/certs/tls.key diff --git a/tests/templates/kuttl/pod_overrides/helm-bitnami-eventlog-minio-values.yaml.j2 b/tests/templates/kuttl/overrides/helm-bitnami-eventlog-minio-values.yaml.j2 similarity index 100% rename from tests/templates/kuttl/pod_overrides/helm-bitnami-eventlog-minio-values.yaml.j2 rename to tests/templates/kuttl/overrides/helm-bitnami-eventlog-minio-values.yaml.j2 diff --git a/tests/templates/kuttl/pod_overrides/helm-bitnami-minio-values.yaml b/tests/templates/kuttl/overrides/helm-bitnami-minio-values.yaml similarity index 100% rename from tests/templates/kuttl/pod_overrides/helm-bitnami-minio-values.yaml rename to tests/templates/kuttl/overrides/helm-bitnami-minio-values.yaml diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index aada4df4..3302a4ec 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -31,7 +31,7 @@ tests: - spark - s3-use-tls - openshift - - name: pod_overrides + - name: overrides dimensions: - spark - openshift From 250e22c6611aa51a6c261848ac7a5e4539d8624d Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 15 Aug 2024 16:01:33 +0200 Subject: [PATCH 03/12] WIP history server tests --- rust/crd/src/history.rs | 61 +++++++++++ .../src/history/history_controller.rs | 102 ++++++++++++++---- .../06-deploy-history-server.yaml.j2 | 2 + 3 files changed, 145 insertions(+), 20 deletions(-) diff --git a/rust/crd/src/history.rs b/rust/crd/src/history.rs index 6d856157..17541e0b 100644 --- a/rust/crd/src/history.rs +++ b/rust/crd/src/history.rs @@ -363,3 +363,64 @@ impl Configuration for HistoryConfigFragment { Ok(BTreeMap::new()) } } + +#[cfg(test)] +mod test { + use super::*; + use indoc::indoc; + + #[test] + pub fn test_env_overrides() { + let input = indoc! {r#" + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkHistoryServer + metadata: + name: spark-history + spec: + image: + productVersion: 3.5.1 + logFileDirectory: + s3: + prefix: eventlogs/ + bucket: + reference: spark-history-s3-bucket + nodes: + envOverrides: + TEST_SPARK_HIST_VAR: ROLE + roleGroups: + default: + replicas: 1 + config: + cleaner: true + envOverrides: + TEST_SPARK_HIST_VAR: ROLEGROUP + "#}; + + let deserializer = serde_yaml::Deserializer::from_str(input); + let history: SparkHistoryServer = + serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); + + assert_eq!( + Some(&"ROLE".to_string()), + history + .spec + .nodes + .config + .env_overrides + .get("TEST_SPARK_HIST_VAR") + ); + assert_eq!( + Some(&"ROLEGROUP".to_string()), + history + .spec + .nodes + .role_groups + .get("default") + .unwrap() + .config + .env_overrides + .get("TEST_SPARK_HIST_VAR") + ); + } +} diff --git a/rust/operator-binary/src/history/history_controller.rs b/rust/operator-binary/src/history/history_controller.rs index b7582f56..4ea893c7 100644 --- a/rust/operator-binary/src/history/history_controller.rs +++ b/rust/operator-binary/src/history/history_controller.rs @@ -440,6 +440,16 @@ fn build_stateful_set( ..PodSecurityContext::default() }); + let role_group = shs + .rolegroup(rolegroupref) + .with_context(|_| CannotRetrieveRoleGroupSnafu)?; + + let merged_env_vars = env_vars( + s3_log_dir, + shs.role().config.clone().env_overrides, + role_group.config.env_overrides, + ); + let container_name = "spark-history"; let container = ContainerBuilder::new(container_name) .context(InvalidContainerNameSnafu)? @@ -449,7 +459,7 @@ fn build_stateful_set( .args(command_args(s3_log_dir)) .add_container_port("http", 18080) .add_container_port("metrics", METRICS_PORT.into()) - .add_env_vars(env_vars(s3_log_dir)) + .add_env_vars(merged_env_vars) .add_volume_mounts(s3_log_dir.volume_mounts()) .add_volume_mount(VOLUME_MOUNT_NAME_CONFIG, VOLUME_MOUNT_PATH_CONFIG) .add_volume_mount(VOLUME_MOUNT_NAME_LOG_CONFIG, VOLUME_MOUNT_PATH_LOG_CONFIG) @@ -670,21 +680,33 @@ fn command_args(s3logdir: &S3LogDir) -> Vec { vec![String::from("-c"), command.join(" && ")] } -fn env_vars(s3logdir: &S3LogDir) -> Vec { - let mut vars: Vec = vec![]; +fn env_vars( + s3logdir: &S3LogDir, + role_env_overrides: HashMap, + role_group_env_overrides: HashMap, +) -> Vec { + // Maps env var name to env var object. This allows env_overrides to work + // as expected (i.e. users can override the env var value). + let mut vars: BTreeMap = BTreeMap::new(); // This env var prevents the history server from detaching itself from the // start script because this leads to the Pod terminating immediately. - vars.push(EnvVar { - name: "SPARK_NO_DAEMONIZE".to_string(), - value: Some("true".into()), - value_from: None, - }); - vars.push(EnvVar { - name: "SPARK_DAEMON_CLASSPATH".to_string(), - value: Some("/stackable/spark/extra-jars/*".into()), - value_from: None, - }); + vars.insert( + "SPARK_NO_DAEMONIZE".to_string(), + EnvVar { + name: "SPARK_NO_DAEMONIZE".to_string(), + value: Some("true".into()), + value_from: None, + }, + ); + vars.insert( + "SPARK_DAEMON_CLASSPATH".to_string(), + EnvVar { + name: "SPARK_DAEMON_CLASSPATH".to_string(), + value: Some("/stackable/spark/extra-jars/*".into()), + value_from: None, + }, + ); let mut history_opts = vec![ format!("-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}"), @@ -693,6 +715,8 @@ fn env_vars(s3logdir: &S3LogDir) -> Vec { ), format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/config.yaml") ]; + + // if TLS is enabled build truststore if tlscerts::tls_secret_name(&s3logdir.bucket.connection).is_some() { history_opts.extend(vec![ format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}/truststore.p12"), @@ -701,13 +725,51 @@ fn env_vars(s3logdir: &S3LogDir) -> Vec { ]); } - vars.push(EnvVar { - name: "SPARK_HISTORY_OPTS".to_string(), - value: Some(history_opts.join(" ")), - value_from: None, - }); - // if TLS is enabled build truststore - vars + vars.insert( + "SPARK_HISTORY_OPTS".to_string(), + EnvVar { + name: "SPARK_HISTORY_OPTS".to_string(), + value: Some(history_opts.join(" ")), + value_from: None, + }, + ); + + // apply the role overrides + let mut role_envs = role_env_overrides + .into_iter() + .map(|env_var| { + ( + env_var.0.clone(), + EnvVar { + name: env_var.0.clone(), + value: Some(env_var.1), + value_from: None, + }, + ) + }) + .collect(); + + vars.append(&mut role_envs); + + // apply the role-group overrides + let mut role_group_envs = role_group_env_overrides + .into_iter() + .map(|env_var| { + ( + env_var.0.clone(), + EnvVar { + name: env_var.0.clone(), + value: Some(env_var.1), + value_from: None, + }, + ) + }) + .collect(); + + vars.append(&mut role_group_envs); + + // convert to Vec + vars.into_values().collect() } fn labels<'a, T>( diff --git a/tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 b/tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 index 965dd18d..3fb5dbff 100644 --- a/tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 +++ b/tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 @@ -45,6 +45,8 @@ spec: replicas: 1 config: cleaner: true + envOverrides: + TEST_SPARK_HIST_VAR_0: ORIGINAL podOverrides: spec: containers: From 7a84f80cd2b9c9b4e30c66ae71c08922c28c0b56 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 15 Aug 2024 17:10:59 +0200 Subject: [PATCH 04/12] working test for history server --- rust/crd/src/history.rs | 2 +- .../kuttl/overrides/06-deploy-history-server.yaml.j2 | 6 +++++- tests/templates/kuttl/overrides/07-assert.yaml | 10 ++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 tests/templates/kuttl/overrides/07-assert.yaml diff --git a/rust/crd/src/history.rs b/rust/crd/src/history.rs index 17541e0b..adbcaba7 100644 --- a/rust/crd/src/history.rs +++ b/rust/crd/src/history.rs @@ -370,7 +370,7 @@ mod test { use indoc::indoc; #[test] - pub fn test_env_overrides() { + pub fn test_env() { let input = indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 diff --git a/tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 b/tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 index 3fb5dbff..8a357c14 100644 --- a/tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 +++ b/tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 @@ -40,13 +40,17 @@ spec: # For possible properties see: https://spark.apache.org/docs/latest/monitoring.html#spark-history-server-configuration-options #sparkConf: nodes: + envOverrides: + TEST_SPARK_HIST_VAR_ROLE: ROLE + TEST_SPARK_HIST_VAR_FROM_RG: ROLE roleGroups: default: replicas: 1 config: cleaner: true envOverrides: - TEST_SPARK_HIST_VAR_0: ORIGINAL + TEST_SPARK_HIST_VAR_FROM_RG: ROLEGROUP + TEST_SPARK_HIST_VAR_RG: ROLEGROUP podOverrides: spec: containers: diff --git a/tests/templates/kuttl/overrides/07-assert.yaml b/tests/templates/kuttl/overrides/07-assert.yaml new file mode 100644 index 00000000..97fef44d --- /dev/null +++ b/tests/templates/kuttl/overrides/07-assert.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +commands: + - script: | + POD=$(kubectl -n $NAMESPACE get pod -l app.kubernetes.io/instance=spark-history -o name | head -n 1 | sed -e 's#pod/##') + kubectl -n $NAMESPACE get pod $POD -o yaml | yq '.spec.containers[0].env[] | select (.name == "TEST_SPARK_HIST_VAR_ROLE").value' | grep 'ROLE' + kubectl -n $NAMESPACE get pod $POD -o yaml | yq '.spec.containers[0].env[] | select (.name == "TEST_SPARK_HIST_VAR_RG").value' | grep 'ROLEGROUP' + kubectl -n $NAMESPACE get pod $POD -o yaml | yq '.spec.containers[0].env[] | select (.name == "TEST_SPARK_HIST_VAR_FROM_RG").value' | grep 'ROLEGROUP' From 2f2f86e5603040daaa95a320368bf0469c9aa328 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 15 Aug 2024 17:18:30 +0200 Subject: [PATCH 05/12] changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 737f3813..bfd17117 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,12 @@ All notable changes to this project will be documented in this file. - `volumes` - `volumeMounts` +### Fixed + +- Fix envOverrides for spark cluster and history server ([#451]). + [#450]: https://github.com/stackabletech/spark-k8s-operator/pull/450 +[#451]: https://github.com/stackabletech/spark-k8s-operator/pull/451 ## [24.7.0] - 2024-07-24 From 944bf36ba7defa5b95c8df0d548a47ffee7f5e96 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Fri, 16 Aug 2024 08:32:17 +0200 Subject: [PATCH 06/12] Update CHANGELOG.md Co-authored-by: Sebastian Bernauer --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfd17117..17960ea0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ All notable changes to this project will be documented in this file. ### Fixed -- Fix envOverrides for spark cluster and history server ([#451]). +- Fix `envOverrides` for SparkApplication and SparkHistoryServer ([#451]). [#450]: https://github.com/stackabletech/spark-k8s-operator/pull/450 [#451]: https://github.com/stackabletech/spark-k8s-operator/pull/451 From dc2f98f216ef9d60feae95f8a27c89252e870d46 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Fri, 16 Aug 2024 08:33:54 +0200 Subject: [PATCH 07/12] Update rust/crd/src/lib.rs Co-authored-by: Sebastian Bernauer --- rust/crd/src/lib.rs | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 30ff03e1..0bdfc051 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -728,33 +728,34 @@ impl SparkApplication { } pub fn merged_env(&self, role: SparkApplicationRole, env: &[EnvVar]) -> Vec { - // use a BTree internally to enable replacement of existing keys - let mut env_vars: BTreeMap = BTreeMap::new(); - - for e in env { - env_vars.insert(e.clone().name, e.to_owned()); - } - - if let Some(env_map) = match role { - SparkApplicationRole::Submit => self.spec.job.clone().map(|j| j.env_overrides), - SparkApplicationRole::Driver => self.spec.driver.clone().map(|d| d.env_overrides), + // Use a BTreeMap internally to enable replacement of existing keys + let mut env: BTreeMap<&String, EnvVar> = env + .iter() + .map(|env_var| (&env_var.name, env_var.clone())) + .collect(); + + // Merge the role-specific envOverrides on top + let role_envs = match role { + SparkApplicationRole::Submit => self.spec.job.as_ref().map(|j| &j.env_overrides), + SparkApplicationRole::Driver => self.spec.driver.as_ref().map(|d| &d.env_overrides), SparkApplicationRole::Executor => { - self.spec.executor.clone().map(|r| r.config.env_overrides) + self.spec.executor.as_ref().map(|e| &e.config.env_overrides) } - } { - for (k, v) in env_map { - env_vars.insert( - k.clone(), + }; + if let Some(role_envs) = role_envs { + env.extend(role_envs.iter().map(|(k, v)| { + ( + k, EnvVar { - name: k, - value: Some(v), - value_from: None, + name: k.clone(), + value: Some(v.clone()), + ..Default::default() }, - ); - } + ) + })) } - env_vars.into_values().collect() + env.into_values().collect() } pub fn validated_role_config( From 6f4b9da7528850fdea49b95940a3eae8490ab022 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Fri, 16 Aug 2024 08:34:16 +0200 Subject: [PATCH 08/12] Update rust/operator-binary/src/history/history_controller.rs Co-authored-by: Sebastian Bernauer --- rust/operator-binary/src/history/history_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/history/history_controller.rs b/rust/operator-binary/src/history/history_controller.rs index 4ea893c7..e5e1d851 100644 --- a/rust/operator-binary/src/history/history_controller.rs +++ b/rust/operator-binary/src/history/history_controller.rs @@ -446,7 +446,7 @@ fn build_stateful_set( let merged_env_vars = env_vars( s3_log_dir, - shs.role().config.clone().env_overrides, + shs.role().config.env_overrides.clone(), role_group.config.env_overrides, ); From 248479d1e050be6ebd38297bf58c9a195dc914bb Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 16 Aug 2024 10:24:44 +0200 Subject: [PATCH 09/12] move merge-env logic to history.rs and improve unit test --- rust/crd/src/constants.rs | 1 + rust/crd/src/history.rs | 124 ++++++++++++++++-- .../src/history/history_controller.rs | 120 ++--------------- 3 files changed, 122 insertions(+), 123 deletions(-) diff --git a/rust/crd/src/constants.rs b/rust/crd/src/constants.rs index 20ef676d..7c1be72b 100644 --- a/rust/crd/src/constants.rs +++ b/rust/crd/src/constants.rs @@ -77,3 +77,4 @@ pub const SPARK_DEFAULTS_FILE_NAME: &str = "spark-defaults.conf"; pub const SPARK_CLUSTER_ROLE: &str = "spark-k8s-clusterrole"; pub const SPARK_UID: i64 = 1000; +pub const METRICS_PORT: u16 = 18081; diff --git a/rust/crd/src/history.rs b/rust/crd/src/history.rs index adbcaba7..2cd285d7 100644 --- a/rust/crd/src/history.rs +++ b/rust/crd/src/history.rs @@ -1,8 +1,11 @@ +use crate::s3logdir::S3LogDir; +use crate::tlscerts; use crate::{affinity::history_affinity, constants::*}; use product_config::{types::PropertyNameKind, ProductConfigManager}; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::k8s_openapi::api::core::v1::EnvVar; use stackable_operator::role_utils::RoleGroup; use stackable_operator::{ commons::{ @@ -232,6 +235,93 @@ impl SparkHistoryServer { ) .context(InvalidProductConfigSnafu) } + + pub fn merged_env( + &self, + s3logdir: &S3LogDir, + role_group_env_overrides: HashMap, + ) -> Vec { + // Maps env var name to env var object. This allows env_overrides to work + // as expected (i.e. users can override the env var value). + let mut vars: BTreeMap = BTreeMap::new(); + let role_env_overrides = &self.role().config.env_overrides; + + // This env var prevents the history server from detaching itself from the + // start script because this leads to the Pod terminating immediately. + vars.insert( + "SPARK_NO_DAEMONIZE".to_string(), + EnvVar { + name: "SPARK_NO_DAEMONIZE".to_string(), + value: Some("true".into()), + value_from: None, + }, + ); + vars.insert( + "SPARK_DAEMON_CLASSPATH".to_string(), + EnvVar { + name: "SPARK_DAEMON_CLASSPATH".to_string(), + value: Some("/stackable/spark/extra-jars/*".into()), + value_from: None, + }, + ); + + let mut history_opts = vec![ + format!("-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}"), + format!( + "-Djava.security.properties={VOLUME_MOUNT_PATH_CONFIG}/{JVM_SECURITY_PROPERTIES_FILE}" + ), + format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/config.yaml") + ]; + + // if TLS is enabled build truststore + if tlscerts::tls_secret_name(&s3logdir.bucket.connection).is_some() { + history_opts.extend(vec![ + format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}/truststore.p12"), + format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TLS_STORE_PASSWORD}"), + format!("-Djavax.net.ssl.trustStoreType=pkcs12"), + ]); + } + + vars.insert( + "SPARK_HISTORY_OPTS".to_string(), + EnvVar { + name: "SPARK_HISTORY_OPTS".to_string(), + value: Some(history_opts.join(" ")), + value_from: None, + }, + ); + + // apply the role overrides + let mut role_envs = role_env_overrides.iter().map(|env_var| { + ( + env_var.0.clone(), + EnvVar { + name: env_var.0.clone(), + value: Some(env_var.1.to_owned()), + value_from: None, + }, + ) + }); + + vars.extend(&mut role_envs); + + // apply the role-group overrides + let mut role_group_envs = role_group_env_overrides.into_iter().map(|env_var| { + ( + env_var.0.clone(), + EnvVar { + name: env_var.0.clone(), + value: Some(env_var.1), + value_from: None, + }, + ) + }); + + vars.extend(&mut role_group_envs); + + // convert to Vec + vars.into_values().collect() + } } #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, Display)] @@ -368,6 +458,7 @@ impl Configuration for HistoryConfigFragment { mod test { use super::*; use indoc::indoc; + use stackable_operator::commons::s3::InlinedS3BucketSpec; #[test] pub fn test_env() { @@ -401,17 +492,16 @@ mod test { let history: SparkHistoryServer = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - assert_eq!( - Some(&"ROLE".to_string()), - history - .spec - .nodes - .config - .env_overrides - .get("TEST_SPARK_HIST_VAR") - ); - assert_eq!( - Some(&"ROLEGROUP".to_string()), + let s3_log_dir: S3LogDir = S3LogDir { + bucket: InlinedS3BucketSpec { + bucket_name: None, + connection: None, + }, + prefix: "prefix".to_string(), + }; + + let merged_env = history.merged_env( + &s3_log_dir, history .spec .nodes @@ -420,7 +510,17 @@ mod test { .unwrap() .config .env_overrides - .get("TEST_SPARK_HIST_VAR") + .clone(), + ); + + let env_map: BTreeMap<&str, Option> = merged_env + .iter() + .map(|env_var| (env_var.name.as_str(), env_var.value.clone())) + .collect(); + + assert_eq!( + Some(&Some("ROLEGROUP".to_string())), + env_map.get("TEST_SPARK_HIST_VAR") ); } } diff --git a/rust/operator-binary/src/history/history_controller.rs b/rust/operator-binary/src/history/history_controller.rs index e5e1d851..fed03bb1 100644 --- a/rust/operator-binary/src/history/history_controller.rs +++ b/rust/operator-binary/src/history/history_controller.rs @@ -15,8 +15,7 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, EnvVar, PodSecurityContext, Service, ServiceAccount, ServicePort, - ServiceSpec, + ConfigMap, PodSecurityContext, Service, ServiceAccount, ServicePort, ServiceSpec, }, rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject}, }, @@ -36,14 +35,15 @@ use stackable_operator::{ role_utils::RoleGroupRef, time::Duration, }; +use stackable_spark_k8s_crd::constants::METRICS_PORT; use stackable_spark_k8s_crd::{ constants::{ ACCESS_KEY_ID, APP_NAME, HISTORY_CONTROLLER_NAME, HISTORY_ROLE_NAME, - JVM_SECURITY_PROPERTIES_FILE, LOG4J2_CONFIG_FILE, MAX_SPARK_LOG_FILES_SIZE, OPERATOR_NAME, - SECRET_ACCESS_KEY, SPARK_CLUSTER_ROLE, SPARK_DEFAULTS_FILE_NAME, SPARK_IMAGE_BASE_NAME, - SPARK_UID, STACKABLE_TLS_STORE_PASSWORD, STACKABLE_TRUST_STORE, VOLUME_MOUNT_NAME_CONFIG, - VOLUME_MOUNT_NAME_LOG, VOLUME_MOUNT_NAME_LOG_CONFIG, VOLUME_MOUNT_PATH_CONFIG, - VOLUME_MOUNT_PATH_LOG, VOLUME_MOUNT_PATH_LOG_CONFIG, + JVM_SECURITY_PROPERTIES_FILE, MAX_SPARK_LOG_FILES_SIZE, OPERATOR_NAME, SECRET_ACCESS_KEY, + SPARK_CLUSTER_ROLE, SPARK_DEFAULTS_FILE_NAME, SPARK_IMAGE_BASE_NAME, SPARK_UID, + STACKABLE_TRUST_STORE, VOLUME_MOUNT_NAME_CONFIG, VOLUME_MOUNT_NAME_LOG, + VOLUME_MOUNT_NAME_LOG_CONFIG, VOLUME_MOUNT_PATH_CONFIG, VOLUME_MOUNT_PATH_LOG, + VOLUME_MOUNT_PATH_LOG_CONFIG, }, history, history::{HistoryConfig, SparkHistoryServer, SparkHistoryServerContainer}, @@ -59,8 +59,6 @@ use stackable_operator::k8s_openapi::DeepMerge; use stackable_operator::logging::controller::ReconcilerError; use strum::{EnumDiscriminants, IntoStaticStr}; -const METRICS_PORT: u16 = 18081; - #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] @@ -444,11 +442,7 @@ fn build_stateful_set( .rolegroup(rolegroupref) .with_context(|_| CannotRetrieveRoleGroupSnafu)?; - let merged_env_vars = env_vars( - s3_log_dir, - shs.role().config.env_overrides.clone(), - role_group.config.env_overrides, - ); + let merged_env = shs.merged_env(s3_log_dir, role_group.config.env_overrides); let container_name = "spark-history"; let container = ContainerBuilder::new(container_name) @@ -459,7 +453,7 @@ fn build_stateful_set( .args(command_args(s3_log_dir)) .add_container_port("http", 18080) .add_container_port("metrics", METRICS_PORT.into()) - .add_env_vars(merged_env_vars) + .add_env_vars(merged_env) .add_volume_mounts(s3_log_dir.volume_mounts()) .add_volume_mount(VOLUME_MOUNT_NAME_CONFIG, VOLUME_MOUNT_PATH_CONFIG) .add_volume_mount(VOLUME_MOUNT_NAME_LOG_CONFIG, VOLUME_MOUNT_PATH_LOG_CONFIG) @@ -487,10 +481,6 @@ fn build_stateful_set( let mut pod_template = pb.build_template(); pod_template.merge_from(shs.role().config.pod_overrides.clone()); - let role_group = shs - .rolegroup(rolegroupref) - .with_context(|_| CannotRetrieveRoleGroupSnafu)?; - pod_template.merge_from(role_group.config.pod_overrides); Ok(StatefulSet { @@ -680,98 +670,6 @@ fn command_args(s3logdir: &S3LogDir) -> Vec { vec![String::from("-c"), command.join(" && ")] } -fn env_vars( - s3logdir: &S3LogDir, - role_env_overrides: HashMap, - role_group_env_overrides: HashMap, -) -> Vec { - // Maps env var name to env var object. This allows env_overrides to work - // as expected (i.e. users can override the env var value). - let mut vars: BTreeMap = BTreeMap::new(); - - // This env var prevents the history server from detaching itself from the - // start script because this leads to the Pod terminating immediately. - vars.insert( - "SPARK_NO_DAEMONIZE".to_string(), - EnvVar { - name: "SPARK_NO_DAEMONIZE".to_string(), - value: Some("true".into()), - value_from: None, - }, - ); - vars.insert( - "SPARK_DAEMON_CLASSPATH".to_string(), - EnvVar { - name: "SPARK_DAEMON_CLASSPATH".to_string(), - value: Some("/stackable/spark/extra-jars/*".into()), - value_from: None, - }, - ); - - let mut history_opts = vec![ - format!("-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}"), - format!( - "-Djava.security.properties={VOLUME_MOUNT_PATH_CONFIG}/{JVM_SECURITY_PROPERTIES_FILE}" - ), - format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/config.yaml") - ]; - - // if TLS is enabled build truststore - if tlscerts::tls_secret_name(&s3logdir.bucket.connection).is_some() { - history_opts.extend(vec![ - format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}/truststore.p12"), - format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TLS_STORE_PASSWORD}"), - format!("-Djavax.net.ssl.trustStoreType=pkcs12"), - ]); - } - - vars.insert( - "SPARK_HISTORY_OPTS".to_string(), - EnvVar { - name: "SPARK_HISTORY_OPTS".to_string(), - value: Some(history_opts.join(" ")), - value_from: None, - }, - ); - - // apply the role overrides - let mut role_envs = role_env_overrides - .into_iter() - .map(|env_var| { - ( - env_var.0.clone(), - EnvVar { - name: env_var.0.clone(), - value: Some(env_var.1), - value_from: None, - }, - ) - }) - .collect(); - - vars.append(&mut role_envs); - - // apply the role-group overrides - let mut role_group_envs = role_group_env_overrides - .into_iter() - .map(|env_var| { - ( - env_var.0.clone(), - EnvVar { - name: env_var.0.clone(), - value: Some(env_var.1), - value_from: None, - }, - ) - }) - .collect(); - - vars.append(&mut role_group_envs); - - // convert to Vec - vars.into_values().collect() -} - fn labels<'a, T>( shs: &'a T, app_version_label: &'a str, From 890c675cc746987f049b6f1d8514e58b98526dfe Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Fri, 16 Aug 2024 10:27:21 +0200 Subject: [PATCH 10/12] Update rust/crd/src/history.rs Co-authored-by: Sebastian Bernauer --- rust/crd/src/history.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crd/src/history.rs b/rust/crd/src/history.rs index 2cd285d7..52606903 100644 --- a/rust/crd/src/history.rs +++ b/rust/crd/src/history.rs @@ -461,7 +461,7 @@ mod test { use stackable_operator::commons::s3::InlinedS3BucketSpec; #[test] - pub fn test_env() { + pub fn test_env_parsing() { let input = indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 From 53f7d59d6dbd879c7c6fbc8737b53d07f87c5a86 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 16 Aug 2024 10:27:50 +0200 Subject: [PATCH 11/12] renamed test --- rust/crd/src/history.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crd/src/history.rs b/rust/crd/src/history.rs index 52606903..4f4e5fb9 100644 --- a/rust/crd/src/history.rs +++ b/rust/crd/src/history.rs @@ -461,7 +461,7 @@ mod test { use stackable_operator::commons::s3::InlinedS3BucketSpec; #[test] - pub fn test_env_parsing() { + pub fn test_env_overrides() { let input = indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 From ad32cefc6f441c9b609e05b8e8592f1f5f1beddd Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 16 Aug 2024 11:20:16 +0200 Subject: [PATCH 12/12] used named tuple for readability --- rust/crd/src/history.rs | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/rust/crd/src/history.rs b/rust/crd/src/history.rs index 4f4e5fb9..882dec2c 100644 --- a/rust/crd/src/history.rs +++ b/rust/crd/src/history.rs @@ -292,12 +292,12 @@ impl SparkHistoryServer { ); // apply the role overrides - let mut role_envs = role_env_overrides.iter().map(|env_var| { + let mut role_envs = role_env_overrides.iter().map(|(env_name, env_value)| { ( - env_var.0.clone(), + env_name.clone(), EnvVar { - name: env_var.0.clone(), - value: Some(env_var.1.to_owned()), + name: env_name.clone(), + value: Some(env_value.to_owned()), value_from: None, }, ) @@ -306,16 +306,19 @@ impl SparkHistoryServer { vars.extend(&mut role_envs); // apply the role-group overrides - let mut role_group_envs = role_group_env_overrides.into_iter().map(|env_var| { - ( - env_var.0.clone(), - EnvVar { - name: env_var.0.clone(), - value: Some(env_var.1), - value_from: None, - }, - ) - }); + let mut role_group_envs = + role_group_env_overrides + .into_iter() + .map(|(env_name, env_value)| { + ( + env_name.clone(), + EnvVar { + name: env_name.clone(), + value: Some(env_value), + value_from: None, + }, + ) + }); vars.extend(&mut role_group_envs);