From c744b5932d59d703fdd9772989c81c95e697b32e Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Wed, 24 Jan 2024 15:12:04 +0000 Subject: [PATCH 01/13] add log collector creds secret to role --- charts/bigdata-spark-watcher/templates/role.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/bigdata-spark-watcher/templates/role.yaml b/charts/bigdata-spark-watcher/templates/role.yaml index 3cf5ceb0..384e4be0 100644 --- a/charts/bigdata-spark-watcher/templates/role.yaml +++ b/charts/bigdata-spark-watcher/templates/role.yaml @@ -26,6 +26,7 @@ rules: - "secrets" resourceNames: - "spotinst-kubernetes-cluster-controller" + - "spot-bigdata-log-collector-creds" verbs: - get - apiGroups: From 0db87ae7fd94b16a338e35fac5fb5b9dc7338e5c Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Wed, 24 Jan 2024 16:47:14 +0000 Subject: [PATCH 02/13] add log collector bucket and enable envs --- charts/bigdata-spark-watcher/values.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/bigdata-spark-watcher/values.yaml b/charts/bigdata-spark-watcher/values.yaml index 62c3efe4..1ad3db2a 100644 --- a/charts/bigdata-spark-watcher/values.yaml +++ b/charts/bigdata-spark-watcher/values.yaml @@ -27,6 +27,10 @@ envVars: value: 6m - name: APP_SYNC_REVERSE_GHOST_GRACE_PERIOD value: 30s + - name: KUBE_EVENT_LOG_COLLECTION_BUCKET + value: "spot-bigdata-logcollector-" + - name: KUBE_EVENT_LOG_COLLECTION_ENABLED + value: "true" - name: CREDS_REFRESH_INTERVAL value: 5m - name: SPARK_APP_FAILED_EXECUTOR_LIMIT From 5cd92b2287567c3ccfca5e5a94b352ec4a19bddf Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Thu, 25 Jan 2024 12:46:32 +0000 Subject: [PATCH 03/13] use argument for enabling event log collection --- charts/bigdata-spark-watcher/templates/deployment.yaml | 2 ++ charts/bigdata-spark-watcher/values.yaml | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/charts/bigdata-spark-watcher/templates/deployment.yaml b/charts/bigdata-spark-watcher/templates/deployment.yaml index 2e0d19a6..eeaceb4f 100644 --- a/charts/bigdata-spark-watcher/templates/deployment.yaml +++ b/charts/bigdata-spark-watcher/templates/deployment.yaml @@ -31,6 +31,8 @@ spec: containers: - name: manager args: + - --enable-event-log-collector + - {{ .Values.eventLogCollectorEnabled}} - --metrics-bind-address - ":8080" - --watch-label diff --git a/charts/bigdata-spark-watcher/values.yaml b/charts/bigdata-spark-watcher/values.yaml index 1ad3db2a..473e7273 100644 --- a/charts/bigdata-spark-watcher/values.yaml +++ b/charts/bigdata-spark-watcher/values.yaml @@ -27,10 +27,8 @@ envVars: value: 6m - name: APP_SYNC_REVERSE_GHOST_GRACE_PERIOD value: 30s - - name: KUBE_EVENT_LOG_COLLECTION_BUCKET - value: "spot-bigdata-logcollector-" - - name: KUBE_EVENT_LOG_COLLECTION_ENABLED - value: "true" + - name: KUBE_EVENT_LOG_COLLECTION_BUCKET_PREFIX + value: "spot-bigdata-logcollector" - name: CREDS_REFRESH_INTERVAL value: 5m - name: SPARK_APP_FAILED_EXECUTOR_LIMIT @@ -77,6 +75,9 @@ resources: cpu: 2000m memory: 2000Mi +eventLogCollectorEnabled: + true + telemetry: enabled: false fluentbit: From 4eb77829e75a4daaf4e764f8116ab4cc0422a960 Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Thu, 25 Jan 2024 13:29:47 +0000 Subject: [PATCH 04/13] add envs for credential files --- charts/bigdata-spark-watcher/values.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/charts/bigdata-spark-watcher/values.yaml b/charts/bigdata-spark-watcher/values.yaml index 473e7273..c374e622 100644 --- a/charts/bigdata-spark-watcher/values.yaml +++ b/charts/bigdata-spark-watcher/values.yaml @@ -29,6 +29,12 @@ envVars: value: 30s - name: KUBE_EVENT_LOG_COLLECTION_BUCKET_PREFIX value: "spot-bigdata-logcollector" + - name: AWS_CREDENTIALS_FILE │ + value: /creds/aws │ + - name: GCP_CREDENTIALS_FILE │ + value: /creds/gcp │ + - name: AZURE_CREDENTIALS_FILE │ + value: /creds/azure - name: CREDS_REFRESH_INTERVAL value: 5m - name: SPARK_APP_FAILED_EXECUTOR_LIMIT From ab3ba862e0d94a0afc6673fe108873c5195846a6 Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Mon, 29 Jan 2024 08:47:58 +0000 Subject: [PATCH 05/13] Update values.yaml --- charts/bigdata-spark-watcher/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/bigdata-spark-watcher/values.yaml b/charts/bigdata-spark-watcher/values.yaml index c374e622..9b5e33cd 100644 --- a/charts/bigdata-spark-watcher/values.yaml +++ b/charts/bigdata-spark-watcher/values.yaml @@ -82,7 +82,7 @@ resources: memory: 2000Mi eventLogCollectorEnabled: - true + false telemetry: enabled: false From a2187b6c63564ff0a3cb480d124092e189a5cb27 Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Tue, 30 Jan 2024 15:56:01 +0000 Subject: [PATCH 06/13] add mount for log collector creds --- charts/bigdata-spark-watcher/templates/deployment.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/charts/bigdata-spark-watcher/templates/deployment.yaml b/charts/bigdata-spark-watcher/templates/deployment.yaml index eeaceb4f..598961e7 100644 --- a/charts/bigdata-spark-watcher/templates/deployment.yaml +++ b/charts/bigdata-spark-watcher/templates/deployment.yaml @@ -123,6 +123,9 @@ spec: mountPath: /var/lib/docker/containers - name: telemetry-aws-credentials mountPath: /root/.aws + - name: spot-bigdata-log-collector-creds + mountPath: /creds + readOnly: true volumes: - name: config configMap: @@ -136,6 +139,9 @@ spec: - name: telemetry-aws-credentials secret: secretName: spot-bigdata-telemetry-creds + - name: spot-bigdata-log-collector-creds + secret: + secretName: spot-bigdata-log-collector-creds {{- end }} {{- with .Values.nodeSelector }} nodeSelector: From 874fff1e5384efb7e75b64b6f1eb43c98a1cfc24 Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Tue, 30 Jan 2024 16:12:20 +0000 Subject: [PATCH 07/13] bump app and chart versions --- charts/bigdata-spark-watcher/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/bigdata-spark-watcher/Chart.yaml b/charts/bigdata-spark-watcher/Chart.yaml index 98b14831..a233648a 100644 --- a/charts/bigdata-spark-watcher/Chart.yaml +++ b/charts/bigdata-spark-watcher/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: bigdata-spark-watcher description: A Helm chart for the Spot Big Data Spark Watcher type: application -version: 0.5.2 -appVersion: 0.4.5 +version: 0.5.4 +appVersion: 0.4.7 home: https://github.com/spotinst/charts icon: https://docs.spot.io/_media/images/spot_mark.png sources: From 6798329bc5e80c1b0c5b15041a4387c8afa9b26f Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Wed, 31 Jan 2024 13:53:29 +0000 Subject: [PATCH 08/13] add list get watch events to the spark watcher cluster role --- charts/bigdata-spark-watcher/templates/role.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/bigdata-spark-watcher/templates/role.yaml b/charts/bigdata-spark-watcher/templates/role.yaml index 384e4be0..20872fa2 100644 --- a/charts/bigdata-spark-watcher/templates/role.yaml +++ b/charts/bigdata-spark-watcher/templates/role.yaml @@ -8,6 +8,7 @@ rules: resources: - "pods" - "namespaces" + - "events" verbs: - get - list From 12077ba79dafd81eddb05bc5fb0ee96fcabe9e49 Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Wed, 31 Jan 2024 16:13:33 +0000 Subject: [PATCH 09/13] allow monitoring kube events --- charts/bigdata-spark-watcher/templates/role.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/charts/bigdata-spark-watcher/templates/role.yaml b/charts/bigdata-spark-watcher/templates/role.yaml index 20872fa2..744dc62a 100644 --- a/charts/bigdata-spark-watcher/templates/role.yaml +++ b/charts/bigdata-spark-watcher/templates/role.yaml @@ -8,11 +8,19 @@ rules: resources: - "pods" - "namespaces" + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: - "events" verbs: - get - list - watch + - create - apiGroups: - "" resources: From 4cbe79a06c4ee3a4380f274d7d2a74e7fb29263f Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Sat, 3 Feb 2024 15:20:31 +0000 Subject: [PATCH 10/13] Update charts/bigdata-spark-watcher/templates/deployment.yaml Co-authored-by: Alex Tarasov <54230036+alextarasov-spot@users.noreply.github.com> --- charts/bigdata-spark-watcher/templates/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/bigdata-spark-watcher/templates/deployment.yaml b/charts/bigdata-spark-watcher/templates/deployment.yaml index 598961e7..aecd751a 100644 --- a/charts/bigdata-spark-watcher/templates/deployment.yaml +++ b/charts/bigdata-spark-watcher/templates/deployment.yaml @@ -31,7 +31,9 @@ spec: containers: - name: manager args: + {{- if .Values.eventLogCollectorEnabled }} - --enable-event-log-collector + {{- end }} - {{ .Values.eventLogCollectorEnabled}} - --metrics-bind-address - ":8080" From 9c5aff5791db4d0c48f6c1026fcd6303fdc883ee Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Sat, 3 Feb 2024 15:20:37 +0000 Subject: [PATCH 11/13] Update charts/bigdata-spark-watcher/templates/deployment.yaml Co-authored-by: Alex Tarasov <54230036+alextarasov-spot@users.noreply.github.com> --- charts/bigdata-spark-watcher/templates/deployment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/bigdata-spark-watcher/templates/deployment.yaml b/charts/bigdata-spark-watcher/templates/deployment.yaml index aecd751a..88542d47 100644 --- a/charts/bigdata-spark-watcher/templates/deployment.yaml +++ b/charts/bigdata-spark-watcher/templates/deployment.yaml @@ -34,7 +34,6 @@ spec: {{- if .Values.eventLogCollectorEnabled }} - --enable-event-log-collector {{- end }} - - {{ .Values.eventLogCollectorEnabled}} - --metrics-bind-address - ":8080" - --watch-label From 073f40ba71a549a063896d60973dda612d72016b Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Wed, 7 Feb 2024 15:18:57 +0000 Subject: [PATCH 12/13] upgrade spark watcher tag --- charts/bigdata-spark-watcher/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/bigdata-spark-watcher/values.yaml b/charts/bigdata-spark-watcher/values.yaml index fb224e6c..ddae8cef 100644 --- a/charts/bigdata-spark-watcher/values.yaml +++ b/charts/bigdata-spark-watcher/values.yaml @@ -8,7 +8,7 @@ image: repository: 066597193667.dkr.ecr.us-east-1.amazonaws.com/private/bigdata-spark-watcher pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: 0.4.6-80ff0942 + tag: 0.4.7-59176f1c imagePullSecrets: - name: spot-bigdata-image-pull From 66fca20023c7a85debcaca43806d940e7cc1c500 Mon Sep 17 00:00:00 2001 From: Sigmar Stefansson Date: Wed, 7 Feb 2024 15:23:05 +0000 Subject: [PATCH 13/13] bump spark watcher chart version --- charts/bigdata-spark-watcher/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/bigdata-spark-watcher/Chart.yaml b/charts/bigdata-spark-watcher/Chart.yaml index a233648a..9c93601e 100644 --- a/charts/bigdata-spark-watcher/Chart.yaml +++ b/charts/bigdata-spark-watcher/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: bigdata-spark-watcher description: A Helm chart for the Spot Big Data Spark Watcher type: application -version: 0.5.4 +version: 0.5.5 appVersion: 0.4.7 home: https://github.com/spotinst/charts icon: https://docs.spot.io/_media/images/spot_mark.png