Merge branch 'main' of https://github.com/bacpop/beebop into bacpop-1…

…86-v9-db

.github/workflows/build_push.yml

@@ -0,0 +1,31 @@
+name: build and push docker images
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - "*"
+env:
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Login to GHCR (GitHub Packages)
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build Server image
+        run: app/server/docker/build
+      - name: Build Proxy image
+        run: proxy/docker/build
+      - name: Docker smoke test
+        run: scripts/docker_smoke_test
+      - name: Push Server image branch ${{ env.BRANCH_NAME }}
+        run: app/server/docker/push
+      - name: Push Proxy image branch ${{ env.BRANCH_NAME }}
+        run: proxy/docker/push

.github/workflows/jestCI.yml

@@ -39,6 +39,8 @@ jobs:
 
   unit_and_integration_backend:
     runs-on: ubuntu-latest
+    env:
+      STORAGE_PATH: ${{ github.workspace }}/storage/dbs
     defaults:
       run:
         working-directory: ./app/server
@@ -49,6 +51,12 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Login to GHCR (GitHub Packages)
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
@@ -60,9 +68,16 @@ jobs:
         run: |
           pwd
           cp ./src/resources/config.json.in.development ./src/resources/config.json
+      - name: Restore storage cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.STORAGE_PATH }}
+          key: dbs-cache-${{ runner.os }}
+          restore-keys: |
+            dbs-cache-
       - name: run all components
         working-directory: .
-        run: ./scripts/run_test server-only
+        run: ./scripts/run_test server-only -mount ${{ env.STORAGE_PATH }}
       - name: Run unit and integration tests
         run: npm run test
       - name: stop all components

.github/workflows/playwrightCI.yml

@@ -13,13 +13,28 @@ jobs:
   playwright-tests:
     timeout-minutes: 60
     runs-on: ubuntu-latest
+    env:
+      STORAGE_PATH: ${{ github.workspace }}/storage/dbs
     strategy:
       fail-fast: false
       matrix:
         browser: [chromium, firefox, webkit]
         node-version: [20.x]
     steps:
       - uses: actions/checkout@v4
+      - name: Login to GHCR (GitHub Packages)
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Restore storage cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.STORAGE_PATH }}
+          key: dbs-cache-${{ runner.os }}
+          restore-keys: |
+            dbs-cache-
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
@@ -32,7 +47,7 @@ jobs:
           cp ./app/server/src/resources/config.json.in.development ./app/server/src/resources/config.json
       - name: Run all components
         working-directory: .
-        run: ./scripts/run_test
+        run: ./scripts/run_test -mount ${{ env.STORAGE_PATH }}
       - name: Install playwright
         working-directory: ./app/client-v2
         run: npx playwright install --with-deps

README.md

@@ -26,7 +26,7 @@ Bring down the app with
 Docker images are built on CI using `./proxy/docker/build`, `./app/server/docker/build`. If you want
 to generate them from changed local sources you can run those same scripts locally to build images.
 
-To target a branch of `beebop_py`, set `API_BRANCH` in `scripts/common`.
+To target a branch of `beebop_py`, set `API_IMAGE` in `scripts/common`.
 
 When running locally in docker, the backend is serving from `beebop_beebop-server_1`, and the front end from the proxy
 container `beebop_proxy_1`.
@@ -77,6 +77,8 @@ You can also run everything outside pm2, by separately running:
 - `./scripts/run_server`
 - `./scripts/run_client`
 
+*Note: If you wish to override the storage volume with a custom bind mount pass in -mount {MOUNT_NAME} into `run_test` script*
+
 ## Config
 Config for the front-end lives in `./app/client/src/settings` and by default webpack (via the vue-cli) will use the config 
 defined in `./app/client/src/settings/development`; this gets overriden by setting an env var called `BUILD_TARGET` - see `./proxy/Dockerfile`.

app/server/docker/common

@@ -1,25 +1,18 @@
 #!/usr/bin/env bash
 set -e
 
+REGISTRY=ghcr.io
 PACKAGE_ROOT=$(realpath $HERE/..)
 PACKAGE_NAME=beebop-server
-PACKAGE_ORG=mrcide
+PACKAGE_ORG=bacpop
 
-# Buildkite doesn't check out a full history from the remote (just the
-# single commit) so you end up with a detached head and git rev-parse
-# doesn't work
-if [ "$BUILDKITE" = "true" ]; then
-    GIT_SHA=${BUILDKITE_COMMIT:0:7}
+GIT_SHA=$(git -C "$PACKAGE_ROOT" rev-parse --short=7 HEAD)
+if [[ -v "BRANCH_NAME" ]]; then
+    GIT_BRANCH=${BRANCH_NAME}
 else
-    GIT_SHA=$(git -C "$PACKAGE_ROOT" rev-parse --short=7 HEAD)
+    GIT_BRANCH=$(git symbolic-ref --short HEAD)
 fi
 
-if [ "$BUILDKITE" = "true" ]; then
-    GIT_BRANCH=$BUILDKITE_BRANCH
-else
-    GIT_BRANCH=$(git -C "$PACKAGE_ROOT" symbolic-ref --short HEAD)
-fi
-
-TAG_SHA="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_SHA}"
-TAG_BRANCH="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}"
-TAG_LATEST="${PACKAGE_ORG}/${PACKAGE_NAME}:latest"
+TAG_SHA="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_SHA}"
+TAG_BRANCH="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}"
+TAG_LATEST="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:latest"

app/server/docker/push

@@ -4,9 +4,6 @@ set -e
 HERE=$(dirname $0)
 . $HERE/common
 
-# In case we switch agents between steps
-[ ! -z $(docker images -q $TAG_SHA) ] || docker pull $TAG_SHA
-
 docker tag $TAG_SHA $TAG_BRANCH
 docker push $TAG_BRANCH
 

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3"
 services:
   proxy:
-    image: mrcide/beebop-proxy:${GIT_SHA}
+    image: ghcr.io/bacpop/beebop-proxy:${GIT_SHA}
     depends_on:
       - beebop-server
     ports:
@@ -10,12 +10,12 @@ services:
     command:
       $HOST beebop-server
   beebop-server:
-    image: mrcide/beebop-server:${GIT_SHA}
+    image: ghcr.io/bacpop/beebop-server:${GIT_SHA}
     depends_on:
       - beebop-py-api
       - beebop-redis
   beebop-py-api:
-    image: mrcide/beebop-py:${API_BRANCH}
+    image: ghcr.io/bacpop/beebop-py:${API_IMAGE}
     depends_on:
       - beebop-redis
     volumes:
@@ -26,7 +26,7 @@ services:
   beebop-redis:
     image: redis:5.0
   beebop-py-worker:
-    image: mrcide/beebop-py:${API_BRANCH}
+    image: ghcr.io/bacpop/beebop-py:${API_IMAGE}
     depends_on:
       - beebop-redis
     command:

proxy/bin/reverse-proxy

@@ -6,7 +6,7 @@ if [ "$#" -eq 2 ]; then
     export SERVER_NAME=$2
 else
     echo "Usage: <hostname> <servername>"
-    echo "e.g. docker run ... mrcide/beebop-proxy:master localhost beebop-server"
+    echo "e.g. docker run ... ghcr.io/bacpop/beebop-proxy:master localhost beebop-server"
     exit 1
 fi
 

proxy/docker/build

@@ -11,4 +11,4 @@ docker build --pull \
 
 # We always push the SHA tagged versions, for debugging if the tests
 # after this step fail
-docker push $TAG_SHA
+docker push $TAG_SHA

proxy/docker/common

@@ -1,25 +1,19 @@
 #!/usr/bin/env bash
 set -e
 
+REGISTRY=ghcr.io
 PACKAGE_ROOT=$(realpath $HERE/..)
 PACKAGE_NAME=beebop-proxy
-PACKAGE_ORG=mrcide
+PACKAGE_ORG=bacpop
 
-# Buildkite doesn't check out a full history from the remote (just the
-# single commit) so you end up with a detached head and git rev-parse
-# doesn't work
-if [ "$BUILDKITE" = "true" ]; then
-    GIT_SHA=${BUILDKITE_COMMIT:0:7}
+GIT_SHA=$(git -C "$PACKAGE_ROOT" rev-parse --short=7 HEAD)
+if [[ -v "BRANCH_NAME" ]]; then
+    GIT_BRANCH=${BRANCH_NAME}
 else
-    GIT_SHA=$(git -C "$PACKAGE_ROOT" rev-parse --short=7 HEAD)
+    GIT_BRANCH=$(git symbolic-ref --short HEAD)
 fi
 
-if [ "$BUILDKITE" = "true" ]; then
-    GIT_BRANCH=$BUILDKITE_BRANCH
-else
-    GIT_BRANCH=$(git -C "$PACKAGE_ROOT" symbolic-ref --short HEAD)
-fi
 
-TAG_SHA="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_SHA}"
-TAG_BRANCH="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}"
-TAG_LATEST="${PACKAGE_ORG}/${PACKAGE_NAME}:latest"
+TAG_SHA="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_SHA}"
+TAG_BRANCH="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}"
+TAG_LATEST="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:latest"

proxy/docker/push

@@ -4,8 +4,6 @@ set -e
 HERE=$(dirname $0)
 . $HERE/common
 
-# In case we switch agents between steps
-[ ! -z $(docker images -q $TAG_SHA) ] || docker pull $TAG_SHA
 
 docker tag $TAG_SHA $TAG_BRANCH
 docker push $TAG_BRANCH

scripts/common

@@ -1,2 +1,9 @@
 #!/usr/bin/env bash
-export API_BRANCH="bacpop-202-fallback-to-refs"
+export API_IMAGE="bacpop-205-fix-network-dev"
+
+NETWORK=beebop_nw
+VOLUME=beebop-storage
+NAME_REDIS=beebop-redis
+NAME_API=beebop-py-api
+NAME_WORKER=beebop-py-worker
+PORT=5000

scripts/parse_mount_arg

@@ -0,0 +1,17 @@
+parse_mount_arg() {
+    local mount=""
+
+    while [ $# -gt 0 ]; do
+        case "$1" in
+            -mount)
+                mount="$2"
+                shift 2
+                ;;
+            *)
+                shift
+                ;;
+        esac
+    done
+
+    echo "$mount"
+}

scripts/run_dependencies

@@ -4,32 +4,35 @@ set -ex
 HERE=$(realpath "$(dirname $0)")
 . $HERE/common
 
-NETWORK=beebop_nw
-VOLUME=beebop-storage
-NAME_REDIS=beebop-redis
-NAME_API=beebop-py-api
-NAME_WORKER=beebop-py-worker
-PORT=5000
-
-docker volume create $VOLUME
-docker run --rm -v $VOLUME:/beebop/storage \
+
+. $HERE/parse_mount_arg
+MOUNT=$(parse_mount_arg "$@")
+
+if [ -z "$MOUNT" ]; then
+       echo "No mount path provided, using default: $VOLUME"
+       MOUNT=$VOLUME
+       docker volume create $VOLUME
+fi
+
+docker network create $NETWORK > /dev/null || /bin/true
+
+docker run --rm -v $MOUNT:/beebop/storage \
        --pull always \
-       mrcide/beebop-py:$API_BRANCH \
+       ghcr.io/bacpop/beebop-py:$API_IMAGE \
        ./scripts/download_databases --refs # remove --refs to download all databases
-docker network create $NETWORK > /dev/null || /bin/true
 
 docker run -d --rm --name $NAME_REDIS --network=$NETWORK -p 6379:6379 redis:5.0
 docker run -d --rm --name $NAME_WORKER --network=$NETWORK \
        --pull always \
        --env=REDIS_HOST="$NAME_REDIS" \
-       -v $VOLUME:/beebop/storage \
-       mrcide/beebop-py:$API_BRANCH rqworker
+       -v $MOUNT:/beebop/storage \
+       ghcr.io/bacpop/beebop-py:$API_IMAGE rqworker
 
 docker run -d --rm --name $NAME_API --network=$NETWORK \
        --pull always \
        --env=REDIS_HOST="$NAME_REDIS" \
        --env=STORAGE_LOCATION="./storage" \
        --env=DBS_LOCATION="./storage/dbs" \
-       -v $VOLUME:/beebop/storage \
+       -v $MOUNT:/beebop/storage \
        -p $PORT:5000 \
-       mrcide/beebop-py:$API_BRANCH
+       ghcr.io/bacpop/beebop-py:$API_IMAGE

scripts/run_docker

@@ -20,5 +20,5 @@ docker cp app/server/src/resources/config.json beebop-beebop-server-1:/app/src/r
 docker cp proxy/ssl/dhparam.pem beebop-proxy-1:/run/proxy/
 docker cp proxy/$SSL_PATH/certificate.pem beebop-proxy-1:/run/proxy/
 docker cp proxy/$SSL_PATH/key.pem beebop-proxy-1:/run/proxy/
-docker run --rm -v beebop_beebop-storage:/beebop/storage mrcide/beebop-py:$API_BRANCH \
+docker run --rm -v beebop_beebop-storage:/beebop/storage ghcr.io/bacpop/beebop-py:$API_IMAGE \
        ./scripts/download_databases --refs

scripts/run_test

@@ -1,7 +1,11 @@
 set -e
 
 HERE=$(realpath "$(dirname $0)")
-. $HERE/run_dependencies
+
+. $HERE/parse_mount_arg
+MOUNT=$(parse_mount_arg "$@")
+. $HERE/run_dependencies -mount $MOUNT
+
 
 
 npm --prefix app/server ci