feat: enhance CI workflows with storage path configuration and cachin…

…g improvements
bacpop · Dec 6, 2024 · 0473946 · 0473946
1 parent cc6778b
commit 0473946
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 25 deletions.
diff --git a/.github/workflows/jestCI.yml b/.github/workflows/jestCI.yml
@@ -39,6 +39,8 @@ jobs:
 
   unit_and_integration_backend:
     runs-on: ubuntu-latest
+    env:
+      STORAGE_PATH: ${{ github.workspace }}/storage/dbs
     defaults:
       run:
         working-directory: ./app/server
@@ -66,27 +68,18 @@ jobs:
         run: |
           pwd
           cp ./src/resources/config.json.in.development ./src/resources/config.json
-      - name: Restore DB Cache
+      - name: Restore storage cache
         uses: actions/cache@v3
         with:
-          path: ${{ github.workspace }}/storage/dbs
-          key: db-cache-${{ runner.os }}
+          path: ${{ env.STORAGE_PATH }}
+          key: dbs-cache-${{ runner.os }}
           restore-keys: |
-            db-cache-
-      - name: run dependencies
+            dbs-cache-
+      - name: run all components
         working-directory: .
-        run: ./scripts/run_dependencies ${{ github.workspace }}/storage/dbs
-      - name: run server
-        working-directory: .
-        run: ./scripts/run_test server-only
+        run: ./scripts/run_test server-only -mount ${{ env.STORAGE_PATH }}
       - name: Run unit and integration tests
         run: npm run test
-
-      - name: Save DB Cache
-        uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/storage/dbs
-          key: db-cache-${{ runner.os }}
       - name: stop all components
         working-directory: .
         run: ./scripts/stop_test server-only
diff --git a/.github/workflows/playwrightCI.yml b/.github/workflows/playwrightCI.yml
@@ -13,6 +13,8 @@ jobs:
   playwright-tests:
     timeout-minutes: 60
     runs-on: ubuntu-latest
+    env:
+      STORAGE_PATH: ${{ github.workspace }}/storage/dbs
     strategy:
       fail-fast: false
       matrix:
@@ -26,6 +28,13 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Restore storage cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.STORAGE_PATH }}
+          key: dbs-cache-${{ runner.os }}
+          restore-keys: |
+            dbs-cache-
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
@@ -38,7 +47,7 @@ jobs:
           cp ./app/server/src/resources/config.json.in.development ./app/server/src/resources/config.json
       - name: Run all components
         working-directory: .
-        run: ./scripts/run_test
+        run: ./scripts/run_test -mount ${{ env.STORAGE_PATH }}
       - name: Install playwright
         working-directory: ./app/client-v2
         run: npx playwright install --with-deps

diff --git a/README.md b/README.md
@@ -77,6 +77,8 @@ You can also run everything outside pm2, by separately running:
 - `./scripts/run_server`
 - `./scripts/run_client`
 
+*Note: If you wish to override the volune with a custom bind mount pass in -mount {MOUNT_NAME} into `run_test` script*
+
 ## Config
 Config for the front-end lives in `./app/client/src/settings` and by default webpack (via the vue-cli) will use the config 
 defined in `./app/client/src/settings/development`; this gets overriden by setting an env var called `BUILD_TARGET` - see `./proxy/Dockerfile`.

diff --git a/scripts/run_dependencies b/scripts/run_dependencies
@@ -4,17 +4,27 @@ set -ex
 HERE=$(realpath "$(dirname $0)")
 . $HERE/common
 
-docker volume create $VOLUME
 
-if [ -z "$1" ]; then
-    echo "No mount path provided, using default: $VOLUME"
-    BIND_MOUNT=$VOLUME
-else
-    BIND_MOUNT=$1
+while [ $# -gt 0 ]; do
+  case "$1" in
+    -mount)
+      MOUNT="$2"
+      shift 2
+      ;;
+    *)
+      shift
+      ;;
+  esac
+done
+if [ -z "$MOUNT" ]; then
+       echo "No mount path provided, using default: $VOLUME"
+       MOUNT=$VOLUME
+       docker volume create $VOLUME
 fi
+
 docker network create $NETWORK > /dev/null || /bin/true
 
-docker run --rm -v $BIND_MOUNT:/beebop/storage \
+docker run --rm -v $MOUNT:/beebop/storage \
        --pull always \
        ghcr.io/bacpop/beebop-py:$API_IMAGE \
        ./scripts/download_databases --refs # remove --refs to download all databases
@@ -23,14 +33,14 @@ docker run -d --rm --name $NAME_REDIS --network=$NETWORK -p 6379:6379 redis:5.0
 docker run -d --rm --name $NAME_WORKER --network=$NETWORK \
        --pull always \
        --env=REDIS_HOST="$NAME_REDIS" \
-       -v $BIND_MOUNT:/beebop/storage \
+       -v $MOUNT:/beebop/storage \
        ghcr.io/bacpop/beebop-py:$API_IMAGE rqworker
 
 docker run -d --rm --name $NAME_API --network=$NETWORK \
        --pull always \
        --env=REDIS_HOST="$NAME_REDIS" \
        --env=STORAGE_LOCATION="./storage" \
        --env=DBS_LOCATION="./storage/dbs" \
-       -v $BIND_MOUNT:/beebop/storage \
+       -v $MOUNT:/beebop/storage \
        -p $PORT:5000 \
        ghcr.io/bacpop/beebop-py:$API_IMAGE
diff --git a/scripts/run_test b/scripts/run_test
@@ -2,6 +2,21 @@ set -e
 
 HERE=$(realpath "$(dirname $0)")
 
+while [ $# -gt 0 ]; do
+  case "$1" in
+    -mount)
+      MOUNT="$2"
+      shift 2
+      ;;
+    *)
+      shift
+      ;;
+  esac
+done
+. $HERE/run_dependencies -mount $MOUNT
+
+
+
 npm --prefix app/server ci
 npm install pm2