Feat: cron deploy pipelines

charts/openmetadata/README.md

@@ -224,6 +224,8 @@ helm install openmetadata open-metadata/openmetadata --values <<path-to-values-f
 | openmetadata.config.smtpConfig.username | string | `Empty String` | SMTP_SERVER_USERNAME |
 | openmetadata.config.upgradeMigrationConfigs.debug | bool | `false` |  |
 | openmetadata.config.upgradeMigrationConfigs.additionalArgs | string | `Empty String` |  |
+| openmetadata.config.deployPipelinesConfig.debug | bool | `false` |   |
+| openmetadata.config.deployPipelinesConfig.additionalArgs | string | `Empty String` |   |
 | openmetadata.config.web.enabled | bool | `true` | |
 | openmetadata.config.web.contentTypeOptions.enabled | bool | `false` | WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED |
 | openmetadata.config.web.csp.enabled | bool | `false` | WEB_CONF_XSS_CSP_ENABLED |
@@ -245,6 +247,7 @@ helm install openmetadata open-metadata/openmetadata --values <<path-to-values-f
 | openmetadata.config.web.permission-policy.enabled | bool | `false` | WEB_CONF_PERMISSION_POLICY_ENABLED |
 | openmetadata.config.web.permission-policy.option | string | `Empty String` | WEB_CONF_PERMISSION_POLICY_OPTION |
 
+
 ## Chart Values
 
 | Key | Type | Default |

charts/openmetadata/templates/_helpers.tpl

@@ -351,3 +351,16 @@ OpenMetadata Configurations Environment Variables*/}}
 {{- end }}
 {{- end }}
 {{- end }}
+
+
+{{/*
+Build the OpenMetadata Deploy Pipelines Command using deployPipelinesConfig */}}
+{{- define "OpenMetadata.buildDeployPipelinesCommand" }}
+  - "/bin/bash"
+  - "-c"
+  {{- if .Values.openmetadata.config.deployPipelinesConfig.debug }}
+  - "/opt/openmetadata/bootstrap/openmetadata-ops.sh -d deploy-pipelines {{ default "" .Values.openmetadata.config.deployPipelinesConfig.additionalArgs }}"
+  {{- else }}
+  - "/opt/openmetadata/bootstrap/openmetadata-ops.sh deploy-pipelines {{ default "" .Values.openmetadata.config.deployPipelinesConfig.additionalArgs }}"
+  {{- end }}
+{{- end }}

charts/openmetadata/templates/cron_deploy_pipeline.yaml

@@ -0,0 +1,133 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cron-deploy-pipelines
+  labels: 
+    {{- include "OpenMetadata.labels" . | indent 4 }}
+    {{- with .Values.deploymentAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 4 }}
+    {{- end }}
+
+spec:
+  suspend: true
+  failedJobsHistoryLimit: 1  
+  successfulJobsHistoryLimit: 1 
+  jobTemplate:
+    metadata:                    
+      name: cron-deploy-pipelines
+    spec:
+      selector:
+      matchLabels: 
+      {{- include "OpenMetadata.selectorLabels" . | nindent 6 }}
+      template:
+        metadata:
+          {{- with .Values.podAnnotations }}
+          annotations:
+            {{- toYaml . | nindent 8 }}
+          {{- end }}
+          labels:
+            {{- include "OpenMetadata.labels" . | indent 8 }}
+        spec:
+          {{- with .Values.imagePullSecrets  }}
+          imagePullSecrets:
+            {{- toYaml . | nindent 8 }}
+          {{- end }}
+          serviceAccountName: {{ include "OpenMetadata.serviceAccountName" . }}
+          {{- if not (.Values.automountServiceAccountToken) }}
+          automountServiceAccountToken: {{ .Values.automountServiceAccountToken}}
+          {{- end }}
+          securityContext:  
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          volumes:
+          {{- include "tplvalues.render" (dict "value" .Values.extraVolumes "context" $) | nindent 16 }}
+          containers:
+            - name: cron-deploy-pipelines
+              securityContext:
+                {{- toYaml .Values.podSecurityContext | nindent 16 }}
+              image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+              imagePullPolicy: {{ .Values.image.pullPolicy }}
+              volumeMounts:
+              {{- with .Values.extraVolumeMounts }}
+                {{- toYaml . | nindent 10 }}
+              {{- end }}
+              command: 
+                {{ include "OpenMetadata.buildDeployPipelinesCommand" . | nindent 16 }}
+              env:
+              {{- include "OpenMetadata.configs" . | nindent 16 }}
+              {{- with .Values.extraEnvs }}
+                {{- toYaml . | nindent 10 }}
+              {{- end }}
+              envFrom:
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-config-secret
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-omd-secret
+                      {{- if .Values.openmetadata.config.database.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-db-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.elasticsearch.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-search-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.authorizer.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-authorizer-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.secretsManager.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-secretsmanager-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.web.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-web-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.authentication.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-authentication-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.eventMonitor.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-eventmonitor-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.pipelineServiceClientConfig.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-pipeline-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.smtpConfig.enableSmtpServer }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-smtp-secret
+                      {{- end }}
+                      {{- if .Values.openmetadata.config.jwtTokenConfiguration.enabled }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" . }}-jwt-secret
+                      {{- end }}
+                      {{- with .Values.openmetadata.config.fernetkey }}
+                      {{- if not .secretRef }}
+                - secretRef:
+                    name: {{ include "OpenMetadata.fullname" $ }}-fernetkey-secret
+                      {{- end }}
+                      {{- end }}
+                      {{- with .Values.envFrom }}
+                        {{- toYaml . | nindent 10 }}
+                      {{- end }}
+              resources:
+                {{- toYaml .Values.resources | nindent 16 }}
+            {{- if .Values.sidecars }}
+              {{- include "tplvalues.render" (dict "value" .Values.sidecars "context" $) | nindent 8 }}
+            {{- end }}
+          {{- with .Values.nodeSelector }}
+          nodeSelector:
+            {{- toYaml . | nindent 8 }}
+          {{- end }}  
+          {{- with .Values.affinity }}
+          affinity:
+            {{- toYaml . | nindent 8 }}
+          {{- end }}
+          {{- with .Values.tolerations }}
+          tolerations:
+            {{- toYaml . | nindent 8 }}
+          {{- end }}                     
+          restartPolicy: OnFailure
+  schedule: "0/5 * * * *"

charts/openmetadata/values.schema.json

@@ -167,6 +167,18 @@
                                 }
                             }
                         },
+                        "deployPipelinesConfig": {
+                            "type": "object",
+                            "additionalProperties": false,
+                            "properties": {
+                                "additionalArgs": {
+                                    "type": "string"
+                                },
+                                "debug": {
+                                    "type": "boolean"
+                                }
+                            }
+                        },
                         "pipelineServiceClientConfig": {
                             "type": "object",
                             "additionalProperties": false,

charts/openmetadata/values.yaml

@@ -12,6 +12,10 @@
       # You can pass the additional argument flags to the openmetadata-ops.sh migrate command
       # Example if you want to force migration runs, use additionalArgs: "--force"
       additionalArgs: ""
+
+    deployPipelinesConfig:
+      debug: false
+      additionalArgs: ""
     # Values can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL
     logLevel: INFO
     clusterName: openmetadata
@@ -410,10 +414,9 @@
   # capabilities:
   #   drop:
   #   - ALL
-  # readOnlyRootFilesystem: true
+# readOnlyRootFilesystem: true
   # runAsNonRoot: true
   # runAsUser: 100
-
 service:
   type: ClusterIP
   port: 8585