Add persistence layer to backend

.pre-commit-config.yaml

@@ -11,13 +11,13 @@ repos:
           - id: end-of-file-fixer
           - id: mixed-line-ending
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.6.9
+      rev: v0.7.1
       hooks:
           - id: ruff
             args: [--fix]
           - id: ruff-format
     - repo: https://github.com/pre-commit/mirrors-mypy
-      rev: v1.11.2
+      rev: v1.13.0
       hooks:
           # See https://github.com/pre-commit/mirrors-mypy/blob/main/.pre-commit-hooks.yaml
           - id: mypy
@@ -31,7 +31,7 @@ repos:
                     --install-types,
                 ]
     - repo: https://github.com/astral-sh/uv-pre-commit
-      rev: 0.4.19
+      rev: 0.4.28
       hooks:
           - id: uv-lock
             name: Lock project dependencies

backend/.gitignore

@@ -0,0 +1,2 @@
+# Development database
+jobq.db

backend/Dockerfile

@@ -3,6 +3,14 @@ ARG PYTHON_VERSION=3.12-slim
 # Build stage
 FROM python:${PYTHON_VERSION} AS build
 
+# Compile bytecode
+# Ref: https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode
+ENV UV_COMPILE_BYTECODE=1
+
+# uv Cache
+# Ref: https://docs.astral.sh/uv/guides/integration/docker/#caching
+ENV UV_LINK_MODE=copy
+
 RUN apt-get update && \
     apt-get install -y git && \
     rm -rf /var/lib/apt/lists/*
@@ -13,20 +21,33 @@ RUN pip install --no-cache-dir --upgrade uv
 ENV SETUPTOOLS_SCM_PRETEND_VERSION_FOR_AAI_JOBQ_SERVER=0.0.0
 
 WORKDIR /code
-COPY ./uv.lock uv.lock
-COPY ./pyproject.toml pyproject.toml
-RUN uv sync --locked
+COPY ./alembic.ini alembic.ini
 
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \ 
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --frozen --no-install-project
 
+COPY ./pyproject.toml ./uv.lock ./alembic.ini /code/
 COPY ./src /code/src
-RUN uv pip install --no-deps .
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync
 
 # Runtime stage
 FROM python:${PYTHON_VERSION}
 
 WORKDIR /code
+COPY scripts/entrypoint.sh /entrypoint.sh
 COPY --chown=nobody:nogroup --from=build /code /code
 
 USER nobody
 
-CMD ["/code/.venv/bin/uvicorn", "jobq_server.__main__:app", "--host", "0.0.0.0", "--port", "8000"]
+ENV PYTHONUNBUFFERED=1
+
+VOLUME ["/data"]
+
+# Provide default path for embedded database
+ENV DB_CONNECTION_STRING="sqlite:////tmp/jobq.db"
+
+ENTRYPOINT ["/entrypoint.sh"]

backend/alembic.ini

@@ -0,0 +1,71 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = src/jobq_server/alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# timezone to use when rendering the date
+# within the migration file as well as the filename.
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

backend/deploy/jobq-server/README.md

@@ -15,19 +15,16 @@ Helm chart for the jobq backend server
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` |  |
+| enableReload | bool | `false` | Enable hot reloading of code inside the container (useful for development) |
 | fullnameOverride | string | `""` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"ghcr.io/aai-institute/jobq-server"` |  |
 | image.tag | string | `""` |  |
 | imagePullSecrets | list | `[]` |  |
-| livenessProbe.httpGet.path | string | `"/health"` |  |
-| livenessProbe.httpGet.port | string | `"http"` |  |
 | nameOverride | string | `""` |  |
 | nodeSelector | object | `{}` |  |
 | podAnnotations | object | `{}` |  |
 | podLabels | object | `{}` |  |
-| readinessProbe.httpGet.path | string | `"/health"` |  |
-| readinessProbe.httpGet.port | string | `"http"` |  |
 | replicaCount | int | `1` |  |
 | resources | object | `{}` |  |
 | service.port | int | `8000` |  |

backend/deploy/jobq-server/templates/deployment.yaml

@@ -6,6 +6,7 @@ metadata:
     {{- include "jobq-server.labels" . | nindent 4 }}
 spec:
   replicas: {{ .Values.replicaCount }}
+  # FIXME: Consider the Recreate deployment strategy to prevent concurrent access to the database
   selector:
     matchLabels:
       {{- include "jobq-server.selectorLabels" . | nindent 6 }}
@@ -39,24 +40,44 @@ spec:
                 - ALL
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          {{- if .Values.enableReload }}
+          args:
+            - "--reload"
+          {{- end }}
           ports:
             - name: http
               containerPort: {{ .Values.service.port }}
               protocol: TCP
+          env:
+            - name: DB_CONNECTION_STRING
+              value: "sqlite:////data/jobq.db"
+            - name: AUTO_MIGRATE
+              value: "false"
           livenessProbe:
-            {{- toYaml .Values.livenessProbe | nindent 12 }}
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 2
           readinessProbe:
-            {{- toYaml .Values.readinessProbe | nindent 12 }}
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 2
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
-          {{- with .Values.volumeMounts }}
           volumeMounts:
+            - name: db-volume
+              mountPath: /data
+          {{- with .Values.volumeMounts }}
             {{- toYaml . | nindent 12 }}
           {{- end }}
-      {{- with .Values.volumes }}
       volumes:
+        - name: db-volume
+          persistentVolumeClaim:
+            claimName: {{ include "jobq-server.fullname" . }}
+        {{- with .Values.volumes }}
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+        {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

backend/deploy/jobq-server/templates/pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+    name: {{ include "jobq-server.fullname" . }}
+spec:
+    accessModes:
+        - ReadWriteOnce
+    resources:
+        requests:
+            storage: 256Mi

backend/deploy/jobq-server/values.yaml

@@ -41,15 +41,6 @@ resources:
     #   cpu: 100m
     #   memory: 128Mi
 
-livenessProbe:
-    httpGet:
-        path: /health
-        port: http
-readinessProbe:
-    httpGet:
-        path: /health
-        port: http
-
 # Additional volumes on the output Deployment definition.
 volumes: []
 # - name: foo
@@ -63,6 +54,9 @@ volumeMounts: []
 #   mountPath: "/etc/foo"
 #   readOnly: true
 
+# -- Enable hot reloading of code inside the container (useful for development)
+enableReload: false
+
 nodeSelector: {}
 
 tolerations: []

backend/pyproject.toml

@@ -16,7 +16,16 @@ maintainers = [
     { name = "Adrian Rumpold", email = "a.rumpold@appliedai-institute.de" },
 ]
 license = { text = "Apache-2.0" }
-dependencies = ["fastapi", "uvicorn", "docker", "kubernetes", "aai-jobq"]
+dependencies = [
+    "fastapi",
+    "uvicorn",
+    "docker",
+    "kubernetes",
+    "aai-jobq",
+    "sqlmodel>=0.0.22",
+    "alembic>=1.13.3",
+    "pydantic-settings>=2.6.0",
+]
 dynamic = ["version"]
 
 [project.optional-dependencies]
@@ -38,6 +47,7 @@ root = ".."
 [tool.ruff]
 extend = "../pyproject.toml"
 src = ["src"]
+extend-exclude = ["src/jobq_server/alembic"]
 
 [tool.mypy]
 ignore_missing_imports = true
@@ -48,6 +58,7 @@ strict_optional = true
 warn_unreachable = true
 show_column_numbers = true
 show_absolute_path = true
+exclude = ["src/jobq_server/alembic"]
 
 [tool.coverage.report]
 exclude_also = ["@overload", "raise NotImplementedError", "if TYPE_CHECKING:"]

backend/scripts/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eux
+
+cd /code
+/code/.venv/bin/alembic upgrade head
+/code/.venv/bin/uvicorn jobq_server.__main__:app --host 0.0.0.0 --port 8000 "$@"

backend/skaffold.yaml

@@ -7,11 +7,19 @@ build:
         - image: ghcr.io/aai-institute/jobq-server
           docker:
               dockerfile: Dockerfile
+          sync:
+              manual:
+                  - src: src/**
+                    dest: /code/src
+    local:
+        useBuildkit: true
 deploy:
     helm:
         releases:
             - name: jobq-server
               chartPath: deploy/jobq-server
+              setValues:
+                  enableReload: true
               valuesFiles:
                   - deploy/jobq-server/values.yaml
               version: 0.1.0

backend/src/jobq_server/__main__.py

@@ -1,22 +1,37 @@
 import logging
 from contextlib import asynccontextmanager
 
-from fastapi import FastAPI
-from kubernetes import config
+import kubernetes.config
+from fastapi import FastAPI, Response
+from sqlmodel import select
 
+from jobq_server.config import settings
+from jobq_server.db import check_migrations, get_engine, upgrade_migrations
 from jobq_server.routers import jobs
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     logging.basicConfig(level=logging.DEBUG)
-    config.load_config()
+
+    # Check if the database schema is up to date
+    needs_migrations = check_migrations()
+    if not needs_migrations:
+        if settings.AUTO_MIGRATE:
+            logging.info("Upgrading database schema")
+            upgrade_migrations()
+        else:
+            logging.error("Database migrations are not up to date. Exiting.")
+            raise SystemExit(1)
+
+    kubernetes.config.load_config()
+
     yield
 
 
 app = FastAPI(
-    title="the jobq cluster workflow management tool backend",
-    description="Backend service for the appliedAI infrastructure product",
+    title="jobq API",
+    description="Backend service API for the jobq workflow engine",
     lifespan=lifespan,
 )
 
@@ -25,7 +40,13 @@ async def lifespan(app: FastAPI):
 
 @app.get("/health", include_in_schema=False)
 async def health():
-    return {"status": "ok"}
+    try:
+        with get_engine().connect() as conn:
+            conn.execute(select(1))
+        return {"status": "ok"}
+    except Exception:
+        logging.error("Database connection failed", exc_info=True)
+        return Response(status_code=503)
 
 
 # URLs to be excluded from Uvicorn access logging