Datasets service k8s deployment

.devcontainer/docker-compose.yml

@@ -12,20 +12,6 @@ services:
  DATABASE_URL: mysql://root:rootpassword@ispyb/ispyb_build
  LOG_LEVEL: DEBUG
 
- opa:
- image: docker.io/openpolicyagent/opa:0.59.0
- restart: unless-stopped
- command: >
- run
- --server
- --config-file /config.yml
- --watch
- /policy
- volumes:
- - ./opa.yml:/config.yml:cached,z
- - ../policy/:/policy:cached,z
- env_file: opa.env
-
  ispyb:
  image: ghcr.io/diamondlightsource/ispyb-database:v3.0.0
  restart: unless-stopped

.github/workflows/helm.yml

@@ -0,0 +1,66 @@
+name: Helm Charts
+
+on:
+ push:
+ pull_request:
+
+jobs:
+ lint:
+ # Deduplicate jobs from pull requests and branch pushes within the same repo.
+ if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout source
+ uses: actions/checkout@v4.1.2
+ with:
+ fetch-depth: 0
+
+ - name: Setup Helm
+ uses: azure/setup-helm@v3.5
+
+ - name: Setup chart-testing
+ uses: helm/chart-testing-action@v2.6.1
+
+ - name: Lint
+ run: >
+ ct
+ lint
+ --target-branch=${{ github.event.repository.default_branch }}
+ --validate-maintainers=false
+
+ build_publish:
+ needs:
+ - lint
+ # Deduplicate jobs from pull requests and branch pushes within the same repo.
+ if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
+ steps:
+ - name: Checkout source
+ uses: actions/checkout@v4.1.2
+
+ - name: Retrieve chart dependencies
+ working-directory: charts/datasets
+ run: helm dependency update
+
+ - name: Package chart
+ working-directory: charts/datasets
+ run: helm package .
+
+ - name: Generate Image Name
+ run: echo IMAGE_REPOSITORY=oci://ghcr.io/$(echo "${{ github.repository }}-chart" | tr '[:upper:]' '[:lower:]' | tr '[_]' '[\-]') >> $GITHUB_ENV
+
+ - name: Log in to GitHub Docker Registry
+ if: github.event_name != 'pull_request'
+ uses: docker/login-action@v3.1.0
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Publish chart
+ if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+ working-directory: charts/datasets
+ run: helm push $(ls datasets-*.tgz) ${{ env.IMAGE_REPOSITORY }}

charts/.gitignore

@@ -0,0 +1,2 @@
+# Chart Bundles
+*.tgz

charts/datasets/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: datasets
+ repository: ""
+ version: 0.1.0
+digest: sha256:bac0b96b8e2519051015ea74ef616cbc76de21a1803714ef9fa25ab896a10f93
+generated: "2024-03-28T11:43:46.834271769Z"

charts/datasets/Chart.yaml

@@ -0,0 +1,11 @@
+apiVersion: v2
+name: datasets
+description: A deployment providing data collected during beamline session as part of the graph federation
+type: application
+
+version: 0.1.0
+
+dependencies:
+ - name: datasets
+ version: 0.1.0
+ condition: datasets.enabled

charts/datasets/charts/datasets/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v2
+name: datasets
+description: A service providing data collected during beamline sessions as part of the graph federation
+type: application
+
+version: 0.1.0
+
+appVersion: 0.1.0-rc5

charts/datasets/charts/datasets/templates/_helpers.tpl

@@ -0,0 +1,73 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "datasets.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "datasets.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "datasets.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "datasets.labels" -}}
+helm.sh/chart: {{ include "datasets.chart" . }}
+{{ include "datasets.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "datasets.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "datasets.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "datasets.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "datasets.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create the database URL string
+*/}}
+{{- define "datasets.databaseURL" -}}
+{{- $host_parts := urlParse .Values.database.host }}
+{{- $raw_user_info := printf "%s:$DATABASE_PASSWORD" .Values.database.user }}
+{{- $url_parts := set $host_parts "userinfo" $raw_user_info }}
+{{- $raw_database_url := urlJoin $url_parts }}
+{{- replace "$DATABASE_PASSWORD" "$(DATABASE_PASSWORD)" $raw_database_url }}
+{{- end }}

charts/datasets/charts/datasets/templates/deployment.yaml

@@ -0,0 +1,65 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "datasets.fullname" . }}
+ labels:
+ {{- include "datasets.labels" . | nindent 4 }}
+spec:
+ replicas: {{ .Values.replicaCount }}
+ selector:
+ matchLabels:
+ {{- include "datasets.selectorLabels" . | nindent 6 }}
+ template:
+ metadata:
+ {{- with .Values.podAnnotations }}
+ annotations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ labels:
+ {{- include "datasets.selectorLabels" . | nindent 8 }}
+ spec:
+ {{- with .Values.imagePullSecrets }}
+ imagePullSecrets:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ serviceAccountName: {{ include "datasets.serviceAccountName" . }}
+ securityContext:
+ {{- toYaml .Values.podSecurityContext | nindent 8 }}
+ containers:
+ - name: {{ .Chart.Name }}
+ securityContext:
+ {{- toYaml .Values.securityContext | nindent 12 }}
+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args: 
+ - serve
+ env:
+ - name: DATABASE_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.database.password.secretName }}
+ key: {{ .Values.database.password.secretKey }}
+ - name: DATABASE_URL
+ value: {{ include "datasets.databaseURL" . }}
+ - name: LOG_LEVEL
+ value: {{ .Values.logLevel }}
+ - name: OTEL_COLLECTOR_URL
+ value: {{ tpl .Values.otelCollectorUrl . }}
+ ports:
+ - name: http
+ containerPort: {{ .Values.service.port }}
+ protocol: TCP
+ resources:
+ {{- toYaml .Values.resources | nindent 12 }}
+ {{- with .Values.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}

charts/datasets/charts/datasets/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "datasets.fullname" . }}
+ labels:
+ {{- include "datasets.labels" . | nindent 4 }}
+spec:
+ type: {{ .Values.service.type }}
+ ports:
+ - port: {{ .Values.service.port }}
+ targetPort: http
+ protocol: TCP
+ name: http
+ selector:
+ {{- include "datasets.selectorLabels" . | nindent 4 }}

charts/datasets/charts/datasets/templates/serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ include "datasets.serviceAccountName" . }}
+ labels:
+ {{- include "datasets.labels" . | nindent 4 }}
+ {{- with .Values.serviceAccount.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+{{- end }}

charts/datasets/charts/datasets/values.yaml

@@ -0,0 +1,47 @@
+nameOverride: ""
+fullnameOverride: ""
+
+image:
+ repository: ghcr.io/diamondlightsource/graph-datasets
+ pullPolicy: Always
+ tag: ""
+
+imagePullSecrets: []
+
+logLevel: Warn
+otelCollectorUrl: ""
+
+database:
+ host: ""
+ user: ""
+ password:
+ secretName: ""
+ secretKey: ""
+
+opa:
+ url: ""
+
+replicaCount: 1
+
+service:
+ type: ClusterIP
+ port: 80
+
+serviceAccount:
+ create: true
+ annotations: {}
+ name: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+
+securityContext: {}
+
+resources: {}
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

charts/datasets/templates/sealed.yaml

@@ -0,0 +1,15 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+ creationTimestamp: null
+ name: ispyb-datasets
+ namespace: graph
+spec:
+ encryptedData:
+ password: AgAkXGCnlnHwdWPkvrDVtkmDaH8okeD4JUQjVjCUttD1KqCaHFuELsfGAU3OrV6egDxZr5N31uhPqnWr3cvqLfBdzqW/Q4gOUGRMqZFx9Lazm+EQfNKieslVqDEuj74/fgdLoUesecO/BD1UykwAKHsJ7Q2YfxamnaMiZRdKPyBqPHPHGYG31LiaS0jHlUkom+KF6P8e8Doe620yEBjbINgOnimBWlsnJ0xDLplZbh8tqajl6BgGUY+1qtA/55HcDcXPbmtiMaC+p85MuXwTN/V1hNGgz5+B9u1Opmush9g247wt7XCD7dgVrCixmJlmAXSNGfNkGERKHG5XK9Lp6mKx84bFVTZNqjyYxtEeNGc96nG8PqH1b54bBKaslgljPCUCOj/PoRrDklrYFVMVv+gs5UpT8EC7RZf7dtjNQbZ71/WOtJO85V0ULnCUgp+gnkjN0RKo+tFFpauiQLDewcJgFUSUnpG5WEFrTZb91Cm46vNZyODc5mRBD1A4KJxye+tS7/K63Kib5xDHhh48PEAoIHYyHGreXAvSmGZ9UYtk1aMAMu9O2AgZalq4s0mPqeGqRHOb5z2cFH/c534zNs3znSNW2G2Ry8e03xt0iAFIwNO7CvHk7vzfT4nRCLCYpn+ppCEub58O6mZ6IGK0dND0YbjD+gaZ8+DCpazVEJ3+b+ZOY31SP9r2T+bBzdIn6JX2ht7tq4MzaklL6cvx/VtS
+ template:
+ metadata:
+ creationTimestamp: null
+ name: ispyb
+ namespace: graph
+

charts/datasets/values.yaml

@@ -0,0 +1,10 @@
+datasets:
+ enabled: true
+ logLevel: Debug
+ database:
+ host: mysql://ispyb-mariadb-galera-headless
+ user: ispyb_ro
+ password:
+ secretName: ispyb-datasets
+ secretKey: password
+ otelCollectorUrl: http://federation-opentelemetry-collector:4317