Merge pull request #96 from mikeiken/dev

Dev

.env-template

@@ -0,0 +1,76 @@
+## SECRETS --==================================
+RABBITMQ_DEFAULT_USER=
+RABBITMQ_DEFAULT_PASS=
+# ---
+POSTGRES_USER=
+POSTGRES_PASSWORD=
+POSTGRES_DB=
+# ---
+REDIS_PASS=
+REDIS_COMMANDER_USER=
+REDIS_COMMANDER_PASS=
+# ---
+DJANGO_SUPER_USER=
+DJANGO_SUPER_PASSWORD=
+DJANGO_SECRET_KEY=
+# ---
+
+## SETTINGS --==================================
+RABBITMQ_WEB_UI_PORT=15672
+RABBITMQ_PORT=5672
+RABBITMQ_HOST=rabbitmq
+RABBITMQ_VHOST=/
+# ---
+POSTGRES_PORT=5432
+# ---
+NGINX_HTTP_PORT=80
+NGINX_HTTPS_PORT=443
+# ---
+REDIS_PORT=6379
+REDIS_MAXMEMORY=256mb
+REDIS_COMMANDER_PORT=8081
+# ---
+PORTAINER_PORT=9000
+# ---
+SWAGGER_PORT=8956
+# ---
+NODEJS_PORT=3000
+# ---
+DJANGO_PORT=8000
+DJANGO_DB_HOST=psql
+# Web debug interface [True/>False<]
+DJANGO_DEBUG=True
+DJANGO_ALLOWED_HOSTS=127.0.0.1,localhost,django
+DJANGO_ASYNC_TIMEOUT_S=30
+# ---
+# Whether to retry failed connections to the broker on startup [>True</False]
+CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP=True
+# Whether to retry failed connections to the broker [>True</False]
+CELERY_BROKER_CONNECTION_RETRY=True
+CELERY_BROKER_CONNECTION_MAX_RETRIES=10
+CELERY_BROKER_HEARTBEAT=10
+CELERY_RESULT_BACKEND=rpc://
+# Late ack means the task messages will be acknowledged after the task has been executed, not right before [>True</False]
+CELERY_TASK_ACKS_LATE=True
+# Run tasks synchronously [True/>False<]
+CELERY_TASK_ALWAYS_EAGER=False
+# Worker process:
+# [
+#     'solo'        - single process
+#    >'prefork'<    - multiple processes (linux only)
+# ]
+CELERY_WORKER_POOL=prefork
+# Restart worker after each task [>4</tasks]
+CELERY_WORKER_MAX_TASKS_PER_CHILD=4
+# Number of worker processes [>2</precesses]
+CELERY_WORKER_CONCURRENCY=2
+# Number of tasks to prefetch [>2</tasks]
+CELERY_WORKER_PREFETCH_MULTIPLIER=2
+# Whether to store the task return values or not (tombstones) [True/>False<]
+CELERY_TASK_IGNORE_RESULT=False
+# Configure task logging [True/>False<]
+CELERY_WORKER_REDIRECT_STDOUTS=False
+# Log level for task logs [DEBUG/>INFO</WARNING/ERROR/CRITICAL]
+CELERY_WORKER_REDIRECT_STDOUTS_LEVEL=INFO
+# Custom logging format for tasks
+CELERY_WORKER_TASK_LOG_FORMAT=[%(asctime)s: %(levelname)s] Task %(task_name)s[%(task_id)s]: %(message)s

.github/workflows/on-commit-dev.yml

@@ -6,10 +6,12 @@ permissions:
 on:
   push:
     branches:
-      - dev
+      - "**"
+      - "!main"
   pull_request:
     branches:
-      - dev
+      - "**"
+      - "!main"
     types: [opened, synchronize, reopened]
   workflow_dispatch:
 
@@ -18,6 +20,52 @@ env:
     PREVIOUS_TAG: 0.0.0
     LATEST_TAG: 0.0.0
     TOML_VERSION: 0.0.0
+    # 
+    RABBITMQ_DEFAULT_USER: test
+    RABBITMQ_DEFAULT_PASS: test
+    POSTGRES_USER: test
+    POSTGRES_PASSWORD: test
+    POSTGRES_DB: test
+    REDIS_PASS: test
+    REDIS_COMMANDER_USER: test
+    REDIS_COMMANDER_PASS: test
+    DJANGO_SUPER_USER: test
+    DJANGO_SUPER_PASSWORD: test
+    DJANGO_SECRET_KEY: django-insecure-test
+    # 
+    RABBITMQ_WEB_UI_PORT: 15672
+    RABBITMQ_PORT: 5672
+    RABBITMQ_HOST: rabbitmq
+    RABBITMQ_VHOST: /
+    POSTGRES_PORT: 5432
+    NGINX_HTTP_PORT: 80
+    NGINX_HTTPS_PORT: 443
+    REDIS_PORT: 6379
+    REDIS_MAXMEMORY: 256mb
+    REDIS_COMMANDER_PORT: 8081
+    PORTAINER_PORT: 9000
+    SWAGGER_PORT: 8956
+    NODEJS_PORT: 3000
+    DJANGO_PORT: 8000
+    DJANGO_DB_HOST: psql
+    DJANGO_DEBUG: True
+    DJANGO_ALLOWED_HOSTS: 127.0.0.1,localhost,django
+    DJANGO_ASYNC_TIMEOUT_S: 30
+    CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP: True
+    CELERY_BROKER_CONNECTION_RETRY: True
+    CELERY_BROKER_CONNECTION_MAX_RETRIES: 10
+    CELERY_BROKER_HEARTBEAT: 10
+    CELERY_RESULT_BACKEND: rpc://
+    CELERY_TASK_ACKS_LATE: True
+    CELERY_TASK_ALWAYS_EAGER: False
+    CELERY_WORKER_POOL: prefork
+    CELERY_WORKER_MAX_TASKS_PER_CHILD: 4
+    CELERY_WORKER_CONCURRENCY: 2
+    CELERY_WORKER_PREFETCH_MULTIPLIER: 2
+    CELERY_TASK_IGNORE_RESULT: False
+    CELERY_WORKER_REDIRECT_STDOUTS: False
+    CELERY_WORKER_REDIRECT_STDOUTS_LEVEL: INFO
+    CELERY_WORKER_TASK_LOG_FORMAT: "[%(asctime)s: %(levelname)s] Task %(task_name)s[%(task_id)s]: %(message)s"
 
 jobs:
   fetch-commit-message:
@@ -58,4 +106,39 @@ jobs:
           sudo apt-get update || true
           sudo apt-get install -y codespell
       - name: Run Spellchecker
-        run: codespell --skip "./frontend/yarn.lock,./papers"
+        run: codespell --skip "./frontend/yarn.lock,./papers"
+
+  pytest:
+    # clone whole repo, run docker-compose-build.yml and run tests
+    runs-on: ubuntu-latest
+    needs: check-no-ci
+    if: needs.check-no-ci.outputs.no_ci == 'false'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Install Docker Compose
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker-compose
+          
+      - name: Login to GitHub Container Registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Create env file from GitHub Environment
+      # use loop to create env file from env variables
+        run: |
+          for key in $(env | grep -E '^[A-Z_]+=' | cut -d= -f1); do
+            echo "$key=${!key}" >> .env
+          done
+
+      - name: Build and run containers
+        run: |
+          docker compose -f docker-compose-build.yml up -d
+          sleep 20  # Wait for containers to be ready
+          
+      - name: Run tests
+        run: docker compose -f docker-compose-build.yml exec -T django pytest

.github/workflows/on-commit-main.yml

@@ -18,6 +18,52 @@ env:
     PREVIOUS_TAG: 0.0.0
     LATEST_TAG: 0.0.0
     TOML_VERSION: 0.0.0
+    # 
+    RABBITMQ_DEFAULT_USER: test
+    RABBITMQ_DEFAULT_PASS: test
+    POSTGRES_USER: test
+    POSTGRES_PASSWORD: test
+    POSTGRES_DB: test
+    REDIS_PASS: test
+    REDIS_COMMANDER_USER: test
+    REDIS_COMMANDER_PASS: test
+    DJANGO_SUPER_USER: test
+    DJANGO_SUPER_PASSWORD: test
+    DJANGO_SECRET_KEY: django-insecure-test
+    # 
+    RABBITMQ_WEB_UI_PORT: 15672
+    RABBITMQ_PORT: 5672
+    RABBITMQ_HOST: rabbitmq
+    RABBITMQ_VHOST: /
+    POSTGRES_PORT: 5432
+    NGINX_HTTP_PORT: 80
+    NGINX_HTTPS_PORT: 443
+    REDIS_PORT: 6379
+    REDIS_MAXMEMORY: 256mb
+    REDIS_COMMANDER_PORT: 8081
+    PORTAINER_PORT: 9000
+    SWAGGER_PORT: 8956
+    NODEJS_PORT: 3000
+    DJANGO_PORT: 8000
+    DJANGO_DB_HOST: psql
+    DJANGO_DEBUG: True
+    DJANGO_ALLOWED_HOSTS: 127.0.0.1,localhost,django
+    DJANGO_ASYNC_TIMEOUT_S: 30
+    CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP: True
+    CELERY_BROKER_CONNECTION_RETRY: True
+    CELERY_BROKER_CONNECTION_MAX_RETRIES: 10
+    CELERY_BROKER_HEARTBEAT: 10
+    CELERY_RESULT_BACKEND: rpc://
+    CELERY_TASK_ACKS_LATE: True
+    CELERY_TASK_ALWAYS_EAGER: False
+    CELERY_WORKER_POOL: prefork
+    CELERY_WORKER_MAX_TASKS_PER_CHILD: 4
+    CELERY_WORKER_CONCURRENCY: 2
+    CELERY_WORKER_PREFETCH_MULTIPLIER: 2
+    CELERY_TASK_IGNORE_RESULT: False
+    CELERY_WORKER_REDIRECT_STDOUTS: False
+    CELERY_WORKER_REDIRECT_STDOUTS_LEVEL: INFO
+    CELERY_WORKER_TASK_LOG_FORMAT: "[%(asctime)s: %(levelname)s] Task %(task_name)s[%(task_id)s]: %(message)s"
 
 jobs:
   fetch-commit-message:
@@ -60,8 +106,45 @@ jobs:
       - name: Run Spellchecker
         run: codespell --skip "./frontend/yarn.lock,./papers"
 
+  pytest:
+    # clone whole repo, run docker-compose-build.yml and run tests
+    runs-on: ubuntu-latest
+    needs: check-no-ci
+    if: needs.check-no-ci.outputs.no_ci == 'false'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Install Docker Compose
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker-compose
+          
+      - name: Login to GitHub Container Registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Create env file from GitHub Environment
+      # use loop to create env file from env variables
+        run: |
+          for key in $(env | grep -E '^[A-Z_]+=' | cut -d= -f1); do
+            echo "$key=${!key}" >> .env
+          done
+
+      - name: Build and run containers
+        run: |
+          docker compose -f docker-compose-build.yml up -d
+          sleep 20  # Wait for containers to be ready
+          
+      - name: Run tests
+        run: docker compose -f docker-compose-build.yml exec -T django pytest
+
   update-release-draft:
-    needs: spellcheck
+    needs: 
+      - spellcheck
+      - pytest
     permissions:
       # write permission is required to create a github release
       contents: write