Changes to merge into 0.8.1

.env.django

@@ -2,6 +2,7 @@
 
 ANTHROPIC_API_KEY=
 OPENAI_API_KEY=
+OPENAI_MODEL=
 LLM_MAX_TOKENS=1024
 
 # === Development ===

.env.docker_build

@@ -0,0 +1 @@
+EMBEDDING_MODEL=all-mpnet-base-v2

.env.uktrade

@@ -0,0 +1,80 @@
+ANALYTICS_LINK=REPLACE_WITH_GITLAB_SECRET
+ANALYTICS_TAG=REPLACE_WITH_GITLAB_SECRET
+
+AUTHBROKER_CLIENT_ID=REPLACE_WITH_GITLAB_SECRET
+AUTHBROKER_CLIENT_SECRET=REPLACE_WITH_GITLAB_SECRET
+AUTHBROKER_URL=https://sso.trade.gov.uk
+
+AWS_ACCESS_KEY=GET_FROM_AWS_ACCESS_PORTAL
+AWS_SECRET_KEY=GET_FROM_AWS_ACCESS_PORTAL
+
+AWS_REGION=eu-west-2
+
+BUCKET_NAME=dbt-default
+
+CHAT_BACKEND=openai
+
+COMPRESSION_ENABLED=false
+
+CONTACT_EMAIL=redbox@businessandtrade.gov.uk
+
+CORE_API_PORT=5002
+EMAIL_BACKEND_TYPE=CONSOLE
+FROM_EMAIL=REPLACE_ME
+
+DEV_MODE=false
+
+DJANGO_SETTINGS_MODULE=redbox_app.settings
+DEBUG=false
+DJANGO_SECRET_KEY=REPLACE_WITH_GITLAB_SECRET
+DJANGO_LOG_LEVEL=DEBUG
+
+ELASTIC__HOST=elasticsearch.internal
+ELASTIC__VERSION=8.11.0
+ELASTIC__USER=elastic
+ELASTIC__PASSWORD=REPLACE_WITH_GITLAB_SECRET
+ELASTIC__PORT=9200
+ELASTIC__SCHEME=http
+
+EMBEDDING_MODEL=all-mpnet-base-v2
+EMBEDDING_BACKEND=sentencetransformers
+EMBEDDING_OPENAI_MODEL=all-mpnet-base-v2
+
+EMBED_QUEUE_NAME=redbox-embedder-queue
+
+ENVIRONMENT=DEV
+
+FILE_EXPIRY_IN_DAYS=30
+
+INGEST_QUEUE_NAME=redbox-ingester-queue
+
+LLM_MAX_TOKENS=4096
+
+LOGIN_METHOD=sso
+
+OBJECT_STORE=s3
+
+OPENAI_API_KEY=REPLACE_WITH_GITLAB_SECRET
+OPENAI_MODEL=gpt-4o-mini
+
+POSTGRES_DB=redbox
+POSTGRES_PASSWORD=REPLACE_WITH_GITLAB_SECRET
+POSTGRES_USER=postgres
+
+REDIS_PORT=6379
+
+USE_STREAMING=false
+
+MAX_SECURITY_CLASSIFICATION=OFFICIAL
+
+SESSION_COOKIE_SECURE=false
+
+SUPERUSER_EMAIL=isobel.daley@digital.trade.gov.uk
+
+REPO_OWNER=uktrade
+
+WORKER_INGEST_MAX_CHUNK_SIZE=600
+WORKER_INGEST_MIN_CHUNK_SIZE=400
+
+ANALYTICS_TAG=REPLACE_WITH_GITLAB_SECRET
+ANALYTICS_LINK=REPLACE_WITH_GITLAB_SECRET

.github/PULL_REQUEST_TEMPLATE.md

@@ -18,3 +18,4 @@
 - [ ] I have added any new ENV vars in all deployed environments
 - [ ] I have tested any code added or changed
 - [ ] I have run integration tests
+- [ ] I have added/updated any DBT specific changes to .env.uktrade

.github/workflows/pull-upstream-release.yml

@@ -0,0 +1,37 @@
+name: Detect upstream release
+
+on:
+  schedule:
+    - cron: '0 8 * * *' # run every day at 8am
+  workflow_dispatch:
+
+jobs:
+  check-upstream-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Fetch upstream release_version
+        run: |
+          git remote add upstream https://github.com/i-dot-ai/redbox.git 
+          git fetch upstream --tags
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check for new release tag
+        id: check_tag
+        run: |
+          latest_tag=$(git describe --tags `git rev-list --tags --max-count=1`)
+          echo "Latest tag: $latest_tag"
+          echo "latest_tag=$latest_tag" >> $GITHUB_ENV
+
+      - name: Create new branch if new release found
+        if: github.event_name == 'schedule'
+        run: |
+          branch_name=upstream/release-${{ env.latest_tag }}
+          echo branch_name: $branch_name
+          git checkout -b $branch_name
+          git push origin $branch_name
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/python-app.yml

@@ -83,7 +83,7 @@ jobs:
         cp .env.test worker/.env
         cd worker
         poetry install
-        poetry run python -m nltk.downloader punkt
+        poetry run python -m nltk.downloader punkt_tab
         poetry run pytest --cov=worker -v --cov-report=term-missing --cov-fail-under=80
 
   test-redbox:

.gitignore

@@ -219,3 +219,7 @@ django_app/tests/staticfiles
 infrastructure/aws/cleanup_lambda/*.zip
 infrastructure/aws/cleanup_lambda/layer/*
 
+
+# chat history files and personal data
+*chat_history_*
+notebooks/evaluation/results

Makefile

@@ -33,7 +33,8 @@ rebuild: stop prune ## Rebuild all images
 	docker compose build --no-cache
 
 .PHONY: test-core-api
-test-core-api: ## Test core-api
+test-core-api: stop ## Test core-api
+	docker compose up -d --wait elasticsearch minio
 	cp .env.test core-api/.env
 	cd core-api && poetry install --with dev && poetry run python -m pytest -m "not ai" --cov=core_api -v --cov-report=term-missing --cov-fail-under=75
 
@@ -124,15 +125,17 @@ docs-build:  ## Build documentation
 	poetry run mkdocs build
 
 # Docker
+AWS_ACCOUNT_ID=REPLACE_ME
+
 AWS_REGION=eu-west-2
-APP_NAME=redbox
+APP_NAME=dbt
 ECR_URL=$(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com
 ECR_REPO_URL=$(ECR_URL)/$(ECR_REPO_NAME)
-IMAGE=$(ECR_REPO_URL):$(IMAGE_TAG)
+IMAGE=latest
 
 ECR_REPO_NAME=$(APP_NAME)
-PREV_IMAGE_TAG=$$(git rev-parse HEAD~1)
-IMAGE_TAG=$$(git rev-parse HEAD)
+PREV_IMAGE_TAG=latest
+IMAGE_TAG=latest
 
 tf_build_args=-var "image_tag=$(IMAGE_TAG)"
 DOCKER_SERVICES=$$(docker compose config --services | grep -v mlflow)
@@ -169,7 +172,7 @@ docker_build: ## Build the docker container
 			PREV_IMAGE="$(ECR_REPO_URL)-$$service:$(PREV_IMAGE_TAG)"; \
 			echo "Pulling previous image: $$PREV_IMAGE"; \
 			docker pull $$PREV_IMAGE; \
-			docker compose build $$service; \
+			DOCKER_DEFAULT_PLATFORM=linux/amd64 docker compose build $$service; \
 		else \
 			echo "Skipping $$service uses default image"; \
 		fi; \

README.md

@@ -13,7 +13,6 @@ Redbox is a retrieval augmented generation (RAG) app that uses GenAI to chat wit
 - **Better retrieval**. Redbox increases organisational memory by indexing documents
 - **Faster, accurate summarisation**. Redbox can summarise reports read months ago, supplement them with current work, and produce a first draft that lets civil servants focus on what they do best
 
-
 https://github.com/i-dot-ai/redbox-copilot/assets/8233643/e7984242-1403-4c93-9e68-03b3f065b38d
 
 # Setup
@@ -90,7 +89,7 @@ redbox/
 
 # Contributing
 
-We welcome contributions to this project. Please see the [CONTRIBUTING.md](./CONTRIBUTING.md) file for more information.
+We welcome contributions to this project. Please see the [CONTRIBUTING.md](./docs/contributing.md) file for more information.
 
 # License
 
@@ -184,9 +183,9 @@ On initial app setup you will need to run `poetry run python manage.py collectst
 
 checkout the `main` branch of the following repos:
 
-* https://github.com/i-dot-ai/redbox
-* https://github.com/i-dot-ai/i-ai-core-infrastructure/
-* https://github.com/i-dot-ai/redbox-copilot-infra-config
+- https://github.com/i-dot-ai/redbox
+- https://github.com/i-dot-ai/i-ai-core-infrastructure/
+- https://github.com/i-dot-ai/redbox-copilot-infra-config
 
 If, and only if, you want to deploy something other than HEAD then replace `var.image_tag` in `infrastructure/aws/ecs.tf` with the hash of the build you want deployed.
 
@@ -197,3 +196,13 @@ cd redbox
 make tf_init
 make tf_apply env=<ENVIRONMENT>
 ```
+
+Local set up for DBT
+
+```
+cp .env.uktrade .env
+docker compose build
+docker compose up -d
+docker compose exec django-app bash
+venv/bin/django-admin createsuperuser
+```

core-api/Dockerfile

@@ -1,17 +1,20 @@
-FROM python:3.12-bookworm as builder
+FROM python:3.12-bookworm AS builder
 
 RUN apt-get update && apt-get install --yes build-essential curl > /dev/null
 
 # Add redbox python package and install it with poetry
 ADD redbox-core/ /app/redbox-core
 ADD core-api/ /app/core-api
 ADD README.md /app/README.md
+ADD download_embedder.py /app/
 
 WORKDIR /app/core-api
 
 RUN pip install poetry
 RUN poetry install
 
+RUN type=cache poetry run python ../download_embedder.py --embedding_model "all-mpnet-base-v2"
+
 # Run FastAPI
 EXPOSE 5002
 CMD ["poetry", "run", "uvicorn", "core_api.app:app", "--host", "0.0.0.0", "--port", "5002"]