From 6741f5431ee899a357b0260b31a3effec5b146af Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 5 Dec 2024 12:25:45 +0530 Subject: [PATCH 1/9] Added cron-deploy-pipeline for Post Upgrade Actions. --- .../templates/cron_deploy_pipeline.yaml | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 charts/openmetadata/templates/cron_deploy_pipeline.yaml diff --git a/charts/openmetadata/templates/cron_deploy_pipeline.yaml b/charts/openmetadata/templates/cron_deploy_pipeline.yaml new file mode 100644 index 0000000..d1a0e66 --- /dev/null +++ b/charts/openmetadata/templates/cron_deploy_pipeline.yaml @@ -0,0 +1,133 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cron-deploy-pipelines + labels: + {{- include "OpenMetadata.labels" . | indent 4 }} + {{- with .Values.deploymentAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + +spec: + suspend: true + failedJobsHistoryLimit: 1 + successfulJobsHistoryLimit: 1 + jobTemplate: + metadata: + name: cron-deploy-pipelines + spec: + selector: + matchLabels: + {{- include "OpenMetadata.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "OpenMetadata.labels" . | indent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "OpenMetadata.serviceAccountName" . }} + {{- if not (.Values.automountServiceAccountToken) }} + automountServiceAccountToken: {{ .Values.automountServiceAccountToken}} + {{- end }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + volumes: + {{- include "tplvalues.render" (dict "value" .Values.extraVolumes "context" $) | nindent 16 }} + containers: + - name: cron-deploy-pipelines + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 16 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + command: + {{ include "OpenMetadata.buildDeployPipelinesCommand" . | nindent 16 }} + env: + {{- include "OpenMetadata.configs" . | nindent 16 }} + {{- with .Values.extraEnvs }} + {{- toYaml . | nindent 10 }} + {{- end }} + envFrom: + - secretRef: + name: {{ include "OpenMetadata.fullname" . }}-config-secret + - secretRef: + name: {{ include "OpenMetadata.fullname" . }}-omd-secret + {{- if .Values.openmetadata.config.database.enabled }} + - secretRef: + name: {{ include "OpenMetadata.fullname" . }}-db-secret + {{- end }} + {{- if .Values.openmetadata.config.elasticsearch.enabled }} + - secretRef: + name: {{ include "OpenMetadata.fullname" . }}-search-secret + {{- end }} + {{- if .Values.openmetadata.config.authorizer.enabled }} + - secretRef: + name: {{ include "OpenMetadata.fullname" . }}-authorizer-secret + {{- end }} + {{- if .Values.openmetadata.config.secretsManager.enabled }} + - secretRef: + name: {{ include "OpenMetadata.fullname" . }}-secretsmanager-secret + {{- end }} + {{- if .Values.openmetadata.config.web.enabled }} + - secretRef: + name: {{ include "OpenMetadata.fullname" . }}-web-secret + {{- end }} + {{- if .Values.openmetadata.config.authentication.enabled }} + - secretRef: + name: {{ include "OpenMetadata.fullname" . }}-authentication-secret + {{- end }} + {{- if .Values.openmetadata.config.eventMonitor.enabled }} + - secretRef: + name: {{ include "OpenMetadata.fullname" . }}-eventmonitor-secret + {{- end }} + {{- if .Values.openmetadata.config.pipelineServiceClientConfig.enabled }} + - secretRef: + name: {{ include "OpenMetadata.fullname" . }}-pipeline-secret + {{- end }} + {{- if .Values.openmetadata.config.smtpConfig.enableSmtpServer }} + - secretRef: + name: {{ include "OpenMetadata.fullname" . }}-smtp-secret + {{- end }} + {{- if .Values.openmetadata.config.jwtTokenConfiguration.enabled }} + - secretRef: + name: {{ include "OpenMetadata.fullname" . }}-jwt-secret + {{- end }} + {{- with .Values.openmetadata.config.fernetkey }} + {{- if not .secretRef }} + - secretRef: + name: {{ include "OpenMetadata.fullname" $ }}-fernetkey-secret + {{- end }} + {{- end }} + {{- with .Values.envFrom }} + {{- toYaml . | nindent 10 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 16 }} + {{- if .Values.sidecars }} + {{- include "tplvalues.render" (dict "value" .Values.sidecars "context" $) | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + restartPolicy: OnFailure + schedule: "0/5 * * * *" From 82d99a32b8d419f12d25437ac0b4f89fcbf4c30a Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 5 Dec 2024 12:27:22 +0530 Subject: [PATCH 2/9] Updated files for cron_deploy_pipeline.yml --- charts/openmetadata/templates/_helpers.tpl | 13 +++++++++++++ charts/openmetadata/values.schema.json | 12 ++++++++++++ charts/openmetadata/values.yaml | 8 ++++++-- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/charts/openmetadata/templates/_helpers.tpl b/charts/openmetadata/templates/_helpers.tpl index 0471087..b8bdefd 100644 --- a/charts/openmetadata/templates/_helpers.tpl +++ b/charts/openmetadata/templates/_helpers.tpl @@ -351,3 +351,16 @@ OpenMetadata Configurations Environment Variables*/}} {{- end }} {{- end }} {{- end }} + + +{{/* +Build the OpenMetadata Deploy Pipelines Command using deployPipelineConfigs */}} +{{- define "OpenMetadata.buildDeployPipelinesCommand" }} + - "/bin/bash" + - "-c" + {{- if .Values.openmetadata.config.deployPipelineConfigs.debug }} + - "/opt/openmetadata/bootstrap/openmetadata-ops.sh -d deploy-pipelines {{ default "" .Values.openmetadata.config.deployPipelineConfigs.additionalArgs }}" + {{- else }} + - "/opt/openmetadata/bootstrap/openmetadata-ops.sh deploy-pipelines {{ default "" .Values.openmetadata.config.deployPipelineConfigs.additionalArgs }}" + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/openmetadata/values.schema.json b/charts/openmetadata/values.schema.json index 1a69878..3f07829 100644 --- a/charts/openmetadata/values.schema.json +++ b/charts/openmetadata/values.schema.json @@ -167,6 +167,18 @@ } } }, + "deployPipelineConfigs": { + "type": "object", + "additionalProperties": false, + "properties": { + "additionalArgs": { + "type": "string" + }, + "debug": { + "type": "boolean" + } + } + }, "pipelineServiceClientConfig": { "type": "object", "additionalProperties": false, diff --git a/charts/openmetadata/values.yaml b/charts/openmetadata/values.yaml index fbdaefc..a3a0dd7 100644 --- a/charts/openmetadata/values.yaml +++ b/charts/openmetadata/values.yaml @@ -13,6 +13,9 @@ openmetadata: # Example if you want to force migration runs, use additionalArgs: "--force" additionalArgs: "" # Values can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL + deployPipelineConfigs: + debug: false + additionalArgs: "" logLevel: INFO clusterName: openmetadata openmetadata: @@ -406,11 +409,12 @@ serviceAccount: automountServiceAccountToken: true podSecurityContext: {} # fsGroup: 2000 -securityContext: {} +# securityContext: {} +securityContext: # capabilities: # drop: # - ALL - # readOnlyRootFilesystem: true + readOnlyRootFilesystem: true # runAsNonRoot: true # runAsUser: 100 From 1212619d132b709a0bf74e1e720cf6a5a018b3eb Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 5 Dec 2024 12:43:55 +0530 Subject: [PATCH 3/9] updated values.yaml --- charts/openmetadata/values.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/charts/openmetadata/values.yaml b/charts/openmetadata/values.yaml index a3a0dd7..75ac13b 100644 --- a/charts/openmetadata/values.yaml +++ b/charts/openmetadata/values.yaml @@ -409,15 +409,13 @@ serviceAccount: automountServiceAccountToken: true podSecurityContext: {} # fsGroup: 2000 -# securityContext: {} -securityContext: +securityContext: {} # capabilities: # drop: # - ALL - readOnlyRootFilesystem: true +# readOnlyRootFilesystem: true # runAsNonRoot: true # runAsUser: 100 - service: type: ClusterIP port: 8585 From 2d40e1679c6e18c19909bbc3a0a8fc10b1656e9b Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 5 Dec 2024 15:35:43 +0530 Subject: [PATCH 4/9] updated required changes --- charts/openmetadata/README.md | 3 +++ charts/openmetadata/templates/_helpers.tpl | 8 ++++---- charts/openmetadata/values.schema.json | 2 +- charts/openmetadata/values.yaml | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/charts/openmetadata/README.md b/charts/openmetadata/README.md index 6483255..8a4613e 100644 --- a/charts/openmetadata/README.md +++ b/charts/openmetadata/README.md @@ -224,6 +224,8 @@ helm install openmetadata open-metadata/openmetadata --values < Date: Thu, 5 Dec 2024 16:32:36 +0530 Subject: [PATCH 5/9] updated sugestions --- charts/openmetadata/templates/_helpers.tpl | 8 ++++---- charts/openmetadata/values.schema.json | 2 +- charts/openmetadata/values.yaml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/charts/openmetadata/templates/_helpers.tpl b/charts/openmetadata/templates/_helpers.tpl index f2a9d66..6ae9f2d 100644 --- a/charts/openmetadata/templates/_helpers.tpl +++ b/charts/openmetadata/templates/_helpers.tpl @@ -354,13 +354,13 @@ OpenMetadata Configurations Environment Variables*/}} {{/* -Build the OpenMetadata Deploy Pipelines Command using deployPipelineConfig */}} +Build the OpenMetadata Deploy Pipelines Command using deployPipelinesConfig */}} {{- define "OpenMetadata.buildDeployPipelinesCommand" }} - "/bin/bash" - "-c" - {{- if .Values.openmetadata.config.deployPipelineConfig.debug }} - - "/opt/openmetadata/bootstrap/openmetadata-ops.sh -d deploy-pipelines {{ default "" .Values.openmetadata.config.deployPipelineConfig.additionalArgs }}" + {{- if .Values.openmetadata.config.deployPipelinesConfig.debug }} + - "/opt/openmetadata/bootstrap/openmetadata-ops.sh -d deploy-pipelines {{ default "" .Values.openmetadata.config.deployPipelinesConfig.additionalArgs }}" {{- else }} - - "/opt/openmetadata/bootstrap/openmetadata-ops.sh deploy-pipelines {{ default "" .Values.openmetadata.config.deployPipelineConfig.additionalArgs }}" + - "/opt/openmetadata/bootstrap/openmetadata-ops.sh deploy-pipelines {{ default "" .Values.openmetadata.config.deployPipelinesConfig.additionalArgs }}" {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/openmetadata/values.schema.json b/charts/openmetadata/values.schema.json index aba891c..25f0e46 100644 --- a/charts/openmetadata/values.schema.json +++ b/charts/openmetadata/values.schema.json @@ -167,7 +167,7 @@ } } }, - "deployPipelineConfig": { + "deployPipelinesConfig": { "type": "object", "additionalProperties": false, "properties": { diff --git a/charts/openmetadata/values.yaml b/charts/openmetadata/values.yaml index afdd2f4..88082bf 100644 --- a/charts/openmetadata/values.yaml +++ b/charts/openmetadata/values.yaml @@ -13,8 +13,8 @@ openmetadata: # Example if you want to force migration runs, use additionalArgs: "--force" additionalArgs: "" # Values can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL - deployPipelineConfig: - debug: false + deployPipelinesConfig: + debug: false additionalArgs: "" logLevel: INFO clusterName: openmetadata From 878ced7db779eb6daafbf3958781bf3bd9c3f797 Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 5 Dec 2024 16:38:21 +0530 Subject: [PATCH 6/9] updated sugestions --- charts/openmetadata/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/openmetadata/README.md b/charts/openmetadata/README.md index 8a4613e..ccc49d1 100644 --- a/charts/openmetadata/README.md +++ b/charts/openmetadata/README.md @@ -224,8 +224,8 @@ helm install openmetadata open-metadata/openmetadata --values < Date: Thu, 5 Dec 2024 16:43:17 +0530 Subject: [PATCH 7/9] updated suggested changes --- charts/openmetadata/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/openmetadata/values.yaml b/charts/openmetadata/values.yaml index 88082bf..e0efbf1 100644 --- a/charts/openmetadata/values.yaml +++ b/charts/openmetadata/values.yaml @@ -12,10 +12,11 @@ openmetadata: # You can pass the additional argument flags to the openmetadata-ops.sh migrate command # Example if you want to force migration runs, use additionalArgs: "--force" additionalArgs: "" - # Values can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL + deployPipelinesConfig: debug: false additionalArgs: "" + # Values can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL logLevel: INFO clusterName: openmetadata openmetadata: From 46530f2788e14df302745b4607b4eeeacb8b0fa8 Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 5 Dec 2024 16:47:53 +0530 Subject: [PATCH 8/9] updated suggested changes --- charts/openmetadata/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/openmetadata/values.yaml b/charts/openmetadata/values.yaml index e0efbf1..26f6d5e 100644 --- a/charts/openmetadata/values.yaml +++ b/charts/openmetadata/values.yaml @@ -12,7 +12,6 @@ openmetadata: # You can pass the additional argument flags to the openmetadata-ops.sh migrate command # Example if you want to force migration runs, use additionalArgs: "--force" additionalArgs: "" - deployPipelinesConfig: debug: false additionalArgs: "" From 1ca10ce99e68b89126d20bbdaa3370ec78c4538c Mon Sep 17 00:00:00 2001 From: Tarun Date: Mon, 9 Dec 2024 11:16:45 +0530 Subject: [PATCH 9/9] fixed merge conflicts --- charts/openmetadata/templates/_helpers.tpl | 4 ++++ charts/openmetadata/values.yaml | 1 + 2 files changed, 5 insertions(+) diff --git a/charts/openmetadata/templates/_helpers.tpl b/charts/openmetadata/templates/_helpers.tpl index e6bd3cb..086d042 100644 --- a/charts/openmetadata/templates/_helpers.tpl +++ b/charts/openmetadata/templates/_helpers.tpl @@ -362,6 +362,10 @@ Build the OpenMetadata Deploy Pipelines Command using deployPipelinesConfig */}} - "/opt/openmetadata/bootstrap/openmetadata-ops.sh -d deploy-pipelines {{ default "" .Values.openmetadata.config.deployPipelinesConfig.additionalArgs }}" {{- else }} - "/opt/openmetadata/bootstrap/openmetadata-ops.sh deploy-pipelines {{ default "" .Values.openmetadata.config.deployPipelinesConfig.additionalArgs }}" + {{- end }} +{{- end }} + + {{/* Build the OpenMetadata Deploy Pipelines Command using reindexConfig */}} {{- define "OpenMetadata.buildReindexCommand" }} diff --git a/charts/openmetadata/values.yaml b/charts/openmetadata/values.yaml index 2255738..0d6d354 100644 --- a/charts/openmetadata/values.yaml +++ b/charts/openmetadata/values.yaml @@ -14,6 +14,7 @@ openmetadata: additionalArgs: "" deployPipelinesConfig: debug: false + additionalArgs: "" reindexConfig: debug: false # You can pass the additional argument flags to the openmetadata-ops.sh reindex command