Datasets service initial structure

.devcontainer/Dockerfile

@@ -0,0 +1,3 @@
+FROM docker.io/library/rust:1.76.0
+
+RUN rustup component add rustfmt clippy

.devcontainer/devcontainer.json

@@ -0,0 +1,23 @@
+{
+ "name": "Datasets",
+ "dockerComposeFile": "docker-compose.yml",
+ "service": "datasets",
+ "customizations": {
+ "vscode": {
+ "extensions": [
+ "ms-kubernetes-tools.vscode-kubernetes-tools",
+ "rust-lang.rust-analyzer",
+ "tamasfe.even-better-toml",
+ "tsandall.opa",
+ "mtxr.sqltools-driver-mysql"
+ ]
+ }
+ },
+ "features": {
+ "ghcr.io/devcontainers/features/common-utils:2.4.2": {
+ "username": "none",
+ "upgradePackages": false
+ }
+ },
+ "workspaceFolder": "/workspace"
+ }

.devcontainer/docker-compose.yml

@@ -0,0 +1,33 @@
+version: "3.8"
+
+services:
+ datasets:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ volumes:
+ - ..:/workspace:cached,z
+ command: sleep infinity
+ environment:
+ DATABASE_URL: mysql://root:rootpassword@ispyb/ispyb_build
+ LOG_LEVEL: DEBUG
+
+ opa:
+ image: docker.io/openpolicyagent/opa:0.59.0
+ restart: unless-stopped
+ command: >
+ run
+ --server
+ --config-file /config.yml
+ --watch
+ /policy
+ volumes:
+ - ./opa.yml:/config.yml:cached,z
+ - ../policy/:/policy:cached,z
+ env_file: opa.env
+
+ ispyb:
+ image: ghcr.io/diamondlightsource/ispyb-database:v3.0.0
+ restart: unless-stopped
+ environment:
+ MARIADB_ROOT_PASSWORD: rootpassword

.devcontainer/opa.yml

@@ -0,0 +1,14 @@
+services:
+ bundler:
+ url: http://authz.diamond.ac.uk
+ credentials:
+ bearer:
+ token: ${BUNDLER_TOKEN}
+
+bundles:
+ permissionables:
+ service: bundler
+ resource: bundle.tar.gz
+ polling:
+ min_delay_seconds: 10
+ max_delay_seconds: 60

.dockerignore

@@ -0,0 +1,19 @@
+# Developer Tooling
+.devcontainer
+.github
+
+# Version Control
+.git
+
+# Container Files
+Dockerfile
+.dockerignore
+
+# OPA Policy
+org-policy/
+
+# Build Artifacts
+target/
+
+# Supplimentary Files
+LICENSE

.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+updates:
+ - package-ecosystem: github-actions
+ directory: /
+ schedule:
+ interval: weekly
+
+ - package-ecosystem: devcontainers
+ directory: "/"
+ schedule:
+ interval: weekly
+
+ - package-ecosystem: cargo
+ directory: /
+ schedule:
+ interval: weekly

.github/workflows/code.yml

@@ -0,0 +1,104 @@
+name: Backend Code
+
+on:
+ push:
+ pull_request:
+
+jobs:
+ lint:
+ # Deduplicate jobs from pull requests and branch pushes within the same repo.
+ if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
+ runs-on: ubuntu-latest
+ services:
+ ispyb:
+ image: ghcr.io/diamondlightsource/ispyb-database:v3.0.0
+ ports:
+ - 3306:3306
+ env:
+ MARIADB_ROOT_PASSWORD: rootpassword
+ options: >
+ --health-cmd "/usr/local/bin/healthcheck.sh --defaults-file=/ispyb/.my.cnf --connect"
+ env:
+ DATABASE_URL: mysql://root:rootpassword@localhost/ispyb_build
+ steps:
+ - name: Checkout source
+ uses: actions/checkout@v4.1.1
+
+ - name: Install dependencies
+ uses: awalsh128/cache-apt-pkgs-action@v1.4.2
+ with:
+ packages: libopencv-dev clang libclang-dev
+
+ - name: Install stable toolchain
+ uses: actions-rs/toolchain@v1.0.7
+ with:
+ toolchain: stable
+ default: true
+
+ - name: Cache Rust Build
+ uses: Swatinem/rust-cache@v2.7.3
+
+ - name: Create models lib file
+ run: |
+ mkdir models/src
+ printf "\n" > models/src/lib.rs
+
+ - name: Check Formatting
+ uses: actions-rs/cargo@v1.0.3
+ with:
+ command: fmt
+ args: >
+ --all
+ --check
+
+ - name: Lint with Clippy
+ uses: actions-rs/cargo@v1.0.3
+ with:
+ command: clippy
+ args: >
+ --no-deps
+ --all-targets
+ --all-features
+ --
+ --deny warnings
+
+ test:
+ # Deduplicate jobs from pull requests and branch pushes within the same repo.
+ if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
+ runs-on: ubuntu-latest
+ services:
+ ispyb:
+ image: ghcr.io/diamondlightsource/ispyb-database:v3.0.0
+ ports:
+ - 3306:3306
+ env:
+ MARIADB_ROOT_PASSWORD: rootpassword
+ options: >
+ --health-cmd "/usr/local/bin/healthcheck.sh --defaults-file=/ispyb/.my.cnf --connect"
+ env:
+ DATABASE_URL: mysql://root:rootpassword@localhost/ispyb_build
+ steps:
+ - name: Checkout source
+ uses: actions/checkout@v4.1.1
+
+ - name: Install dependencies
+ uses: awalsh128/cache-apt-pkgs-action@v1.4.2
+ with:
+ packages: libopencv-dev clang libclang-dev
+
+ - name: Install stable toolchain
+ uses: actions-rs/toolchain@v1.0.7
+ with:
+ toolchain: stable
+ default: true
+
+ - name: Cache Rust Build
+ uses: Swatinem/rust-cache@v2.7.3
+
+ - name: Test
+ uses: actions-rs/cargo@v1.0.3
+ with:
+ command: test
+ args: >
+ --all-targets
+ --all-features

.github/workflows/container.yml

@@ -0,0 +1,63 @@
+name: Container
+
+on:
+ push:
+ pull_request:
+
+jobs:
+ build:
+ # Deduplicate jobs from pull requests and branch pushes within the same repo.
+ if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
+ runs-on: ubuntu-latest
+ services:
+ ispyb:
+ image: ghcr.io/diamondlightsource/ispyb-database:v3.0.0
+ ports:
+ - 3306:3306
+ env:
+ MARIADB_ROOT_PASSWORD: rootpassword
+ options: >
+ --health-cmd "/usr/local/bin/healthcheck.sh --defaults-file=/ispyb/.my.cnf --connect"
+ permissions:
+ contents: read
+ packages: write
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout@v4.1.1
+
+ - name: Generate Image Name
+ run: echo IMAGE_REPOSITORY=ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]' | tr '[_]' '[\-]') >> $GITHUB_ENV
+
+ - name: Log in to GitHub Docker Registry
+ if: github.event_name != 'pull_request'
+ uses: docker/login-action@v3.0.0
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Docker Metadata
+ id: meta
+ uses: docker/metadata-action@v5.5.1
+ with:
+ images: ${{ env.IMAGE_REPOSITORY }}
+ tags: |
+ type=ref,event=tag
+ type=raw,value=latest
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3.1.0
+ with:
+ driver-opts: network=host
+
+ - name: Build Image
+ uses: docker/build-push-action@v5.2.0
+ with:
+ build-args: DATABASE_URL=mysql://root:rootpassword@localhost:3306/ispyb_build
+ target: deploy
+ push: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }}
+ load: ${{ !(github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) }}
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ cache-from: type=gha
+ cache-to: type=gha,mode=max

.github/workflows/devcontainer.yml

@@ -0,0 +1,23 @@
+name: Dev Container CI
+
+on:
+ push:
+ pull_request:
+
+jobs:
+ build:
+ # pull requests are a duplicate of a branch push if within the same repo.
+ if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4.1.1
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3.1.0
+
+ - name: Create .env file
+ run: touch .devcontainer/opa.env
+
+ - name: Build dev container
+ uses: devcontainers/ci@v0.3.1900000347

.gitignore

@@ -0,0 +1,5 @@
+# Secrets
+*.env
+
+# Rust Builds
+/target

.vscode/settings.json

@@ -0,0 +1,18 @@
+{
+ "sqltools.connections": [
+ {
+ "mysqlOptions": {
+ "authProtocol": "default",
+ "enableSsl": "Disabled"
+ },
+ "previewLimit": 50,
+ "server": "ispyb",
+ "port": 3306,
+ "driver": "MariaDB",
+ "name": "mariadb",
+ "database": "ispyb_build",
+ "username": "root",
+ "askForPassword": true
+ }
+ ]
+}