From 4eaa6029d545f22de97d3b0c18441374c0719397 Mon Sep 17 00:00:00 2001 From: Christian Kugler Date: Mon, 30 Dec 2024 15:26:34 +0100 Subject: [PATCH] Add option to create indices from existingConfigmap Currently the Helm Chart supports creating new repositories via an existing ConfigMap. For deploying GraphDB as IaC this is quite useful. Unfortunately, there is no way to create indices at install time. This adds the option to provide a existing ConfigMap via values and create an index for one or multiple repositories. The ConfigMap has to consist out of keys that have the same name as the repository. The value is the SPARQL query. --- README.md | 1 + files/scripts/graphdb.sh | 38 ++++++++++ templates/jobs/_labels.tpl | 4 ++ templates/jobs/job-create-indices.yaml | 99 ++++++++++++++++++++++++++ values.yaml | 10 +++ 5 files changed, 152 insertions(+) create mode 100644 templates/jobs/job-create-indices.yaml diff --git a/README.md b/README.md index 02914495..bafa2e38 100644 --- a/README.md +++ b/README.md @@ -387,6 +387,7 @@ IMPORTANT: This is generated by helm-docs, do not attempt modifying it on hand a | import.volumeMount.volumeClaimTemplate.name | string | `"import"` | | | import.volumeMount.volumeClaimTemplate.spec.accessModes[0] | string | `"ReadWriteOnce"` | | | import.volumeMount.volumeClaimTemplate.spec.resources.requests.storage | string | `"10Gi"` | | +| indices.existingConfigmap | string | `""` | | | ingress.annotations | object | `{}` | | | ingress.className | string | `""` | | | ingress.enabled | bool | `true` | | diff --git a/files/scripts/graphdb.sh b/files/scripts/graphdb.sh index c7a1caf0..961aba01 100755 --- a/files/scripts/graphdb.sh +++ b/files/scripts/graphdb.sh @@ -155,6 +155,44 @@ function cloudBackup { log "Backup ${BACKUP_NAME} completed successfully!" } +function createIndicesFromFiles() { + local node_count=$1 + local indicesConfigsLocation=$2 + local success=true + local response + local repository + + set -v + + waitAllNodes "$node_count" + + for filename in "${indicesConfigsLocation}"/*; do + repository="$(basename "$filename")" + response=$( + curl --connect-timeout 60 --retry 3 --retry-all-errors --retry-delay 10 \ + -H "Authorization: Basic ${GRAPHDB_AUTH_TOKEN}" \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/sparql-update' \ + --data-binary @"${filename}" \ + "http://${GRAPHDB_POD_NAME}-0.${GRAPHDB_SERVICE_NAME}:${GRAPHDB_SERVICE_PORT}/repositories/${repository}/statements" + ) + + if [ -z "$response" ]; then + log "Successfully created index for ${filename}" + else + log "Could not create index for ${filename}, response:" + log "$response" + if ! grep -q "already exists." <<<$response; then + success=false + fi + fi + done + + if [ $success != true ]; then + exit 1 + fi +} + function localBackup() { BACKUP_TIMESTAMP="$(date +'%Y-%m-%d_%H-%M-%S')" BACKUP_NAME="graphdb-backup-${BACKUP_TIMESTAMP}.tar" diff --git a/templates/jobs/_labels.tpl b/templates/jobs/_labels.tpl index 92b728db..b949c2f5 100644 --- a/templates/jobs/_labels.tpl +++ b/templates/jobs/_labels.tpl @@ -30,6 +30,10 @@ Helper functions for labels related to Job and provisioning resources {{- printf "%s-%s" (include "graphdb.fullname" .) "provision-repositories" -}} {{- end -}} +{{- define "graphdb.fullname.job.provision-indices" -}} + {{- printf "%s-%s" (include "graphdb.fullname" .) "provision-indices" -}} +{{- end -}} + {{- define "graphdb.fullname.job.scale-down-cluster" -}} {{- printf "%s-%s" (include "graphdb.fullname" .) "scale-down-cluster" -}} {{- end -}} diff --git a/templates/jobs/job-create-indices.yaml b/templates/jobs/job-create-indices.yaml new file mode 100644 index 00000000..4827e1b2 --- /dev/null +++ b/templates/jobs/job-create-indices.yaml @@ -0,0 +1,99 @@ +{{- if .Values.indices.existingConfigmap }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "graphdb.fullname.job.provision-indices" . }} + namespace: {{ include "graphdb.namespace" . }} + labels: + {{- include "graphdb.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install, post-upgrade, post-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "5" + {{- with .Values.annotations }} + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} +spec: + backoffLimit: {{ .Values.jobs.backoffLimit }} + ttlSecondsAfterFinished: {{ .Values.jobs.ttlSecondsAfterFinished }} + template: + spec: + restartPolicy: Never + automountServiceAccountToken: false + {{- if .Values.jobs.schedulerName }} + schedulerName: {{ .Values.jobs.schedulerName }} + {{- end }} + {{- with .Values.jobs.dnsConfig }} + dnsConfig: {{ toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.jobs.dnsPolicy }} + dnsPolicy: {{ .Values.jobs.dnsPolicy }} + {{- end }} + {{- if .Values.jobs.priorityClassName }} + priorityClassName: {{ .Values.jobs.priorityClassName }} + {{- end }} + {{- with .Values.jobs.nodeSelector }} + nodeSelector: {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- with .Values.jobs.affinity }} + affinity: {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- with .Values.jobs.tolerations }} + tolerations: {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- with .Values.jobs.topologySpreadConstraints }} + topologySpreadConstraints: {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- if or .Values.global.imagePullSecrets .Values.image.pullSecrets }} + imagePullSecrets: + {{- include "graphdb.combinedImagePullSecrets" . | nindent 8 }} + {{- end }} + {{- with .Values.jobs.podSecurityContext }} + securityContext: {{ toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: job-temp + emptyDir: {{ .Values.jobs.persistence.emptyDir | toYaml | nindent 12 }} + - name: indices-config + configMap: + name: {{ .Values.indices.existingConfigmap }} + - name: graphdb-utils + configMap: + name: {{ include "graphdb.fullname.configmap.utils" . }} + containers: + - name: provision-indices + image: {{ include "graphdb.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: GRAPHDB_POD_NAME + value: {{ include "graphdb.fullname" . }} + - name: GRAPHDB_SERVICE_NAME + value: {{ include "graphdb.fullname.service.headless" . }} + - name: GRAPHDB_SERVICE_PORT + value: {{ .Values.headlessService.ports.http | quote }} + - name: GRAPHDB_AUTH_TOKEN + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.security.provisioner.existingSecret .) | default (include "graphdb.fullname.secret.provisioning-user" .) }} + key: {{ .Values.security.provisioner.tokenKey }} + {{- with .Values.jobs.securityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.jobs.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: job-temp + mountPath: /tmp + - name: graphdb-utils + mountPath: /tmp/utils + - name: indices-config + mountPath: /tmp/indices-config + workingDir: /tmp + command: ["bash"] + args: + - "/tmp/utils/graphdb.sh" + - "createIndicesFromFiles" + - "{{ .Values.replicas }}" + - "/tmp/indices-config" +{{- end }} diff --git a/values.yaml b/values.yaml index 4610f037..51b7fe90 100644 --- a/values.yaml +++ b/values.yaml @@ -969,6 +969,16 @@ repositories: # Each key in the existing configmap will be treated as the config.ttl of a repository. existingConfigmap: "" +################################ +# Custom Lucene Configurations # +################################ + +indices: + # Optional configmap containing indices configuration .ttl file(s). + # GraphDB will automatically create indices with the provided repositories configuration files. + # Each key in the existing configmap will be treated as the config.ttl of a repository. + existingConfigmap: "" + ################################################ # Persistent Volume Permissions Configurations # ################################################