From 5f31e250d43c05a6ab9cfafe16500a6b2fe07858 Mon Sep 17 00:00:00 2001 From: Isobel Daley Date: Thu, 4 Jul 2024 15:51:19 +0100 Subject: [PATCH 001/153] feat: make openAI model version configurable --- .env.django | 1 + .env.example | 1 + .env.test | 1 + core_api/src/dependencies.py | 28 +++++++++++++++++++-------- redbox-core/redbox/models/settings.py | 25 ++++++++++++++++-------- 5 files changed, 40 insertions(+), 16 deletions(-) diff --git a/.env.django b/.env.django index 8aa7ddaed..41bb42dd2 100644 --- a/.env.django +++ b/.env.django @@ -2,6 +2,7 @@ ANTHROPIC_API_KEY= OPENAI_API_KEY= +OPENAI_MODEL= LLM_MAX_TOKENS=1024 # === Development === diff --git a/.env.example b/.env.example index 9a78c01e5..ed2f1714a 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,7 @@ ANTHROPIC_API_KEY= OPENAI_API_KEY= +OPENAI_API_KEY= LLM_MAX_TOKENS=1024 # === Development === diff --git a/.env.test b/.env.test index 6fac0ca6e..0bc9191ce 100644 --- a/.env.test +++ b/.env.test @@ -2,6 +2,7 @@ ANTHROPIC_API_KEY= OPENAI_API_KEY= +OPENAI_MODEL= LLM_MAX_TOKENS=1024 # === AI === diff --git a/core_api/src/dependencies.py b/core_api/src/dependencies.py index db3eab95e..d90c8594e 100644 --- a/core_api/src/dependencies.py +++ b/core_api/src/dependencies.py @@ -12,13 +12,16 @@ from langchain_core.retrievers import BaseRetriever from langchain_core.runnables import ConfigurableField from langchain_elasticsearch import ApproxRetrievalStrategy, ElasticsearchStore - -from core_api.src.callbacks import LoggerCallbackHandler -from core_api.src.retriever import AllElasticsearchRetriever, ParameterisedElasticsearchRetriever from redbox.model_db import MODEL_PATH from redbox.models import Settings from redbox.storage import ElasticsearchStorageHandler +from core_api.src.callbacks import LoggerCallbackHandler +from core_api.src.retriever import ( + AllElasticsearchRetriever, + ParameterisedElasticsearchRetriever, +) + logging.basicConfig(level=logging.INFO) log = logging.getLogger() @@ -29,13 +32,17 @@ def get_env() -> Settings: @lru_cache(1) -def get_elasticsearch_client(env: Annotated[Settings, Depends(get_env)]) -> Elasticsearch: +def get_elasticsearch_client( + env: Annotated[Settings, Depends(get_env)] +) -> Elasticsearch: return env.elasticsearch_client() @lru_cache(1) def get_embedding_model(env: Annotated[Settings, Depends(get_env)]) -> Embeddings: - embedding_model = SentenceTransformerEmbeddings(model_name=env.embedding_model, cache_folder=MODEL_PATH) + embedding_model = SentenceTransformerEmbeddings( + model_name=env.embedding_model, cache_folder=MODEL_PATH + ) log.info("Loaded embedding model from environment: %s", env.embedding_model) return embedding_model @@ -72,7 +79,8 @@ def get_vector_store( @lru_cache(1) def get_parameterised_retriever( - env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)] + env: Annotated[Settings, Depends(get_env)], + es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], ) -> BaseRetriever: """Creates an Elasticsearch retriever runnable. @@ -94,14 +102,17 @@ def get_parameterised_retriever( embedding_model=get_embedding_model(env), ).configurable_fields( params=ConfigurableField( - id="params", name="Retriever parameters", description="A dictionary of parameters to use for the retriever." + id="params", + name="Retriever parameters", + description="A dictionary of parameters to use for the retriever.", ) ) @lru_cache(1) def get_all_chunks_retriever( - env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)] + env: Annotated[Settings, Depends(get_env)], + es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], ): return AllElasticsearchRetriever( es_client=es, @@ -120,6 +131,7 @@ def get_llm(env: Annotated[Settings, Depends(get_env)]) -> ChatLiteLLM: streaming=True, openai_key=env.openai_api_key, callbacks=[logger_callback], + model=env.openai_model, ) elif env.azure_openai_api_key is not None: log.info("Creating Azure LLM Client") diff --git a/redbox-core/redbox/models/settings.py b/redbox-core/redbox/models/settings.py index 2716eba79..fb1534980 100644 --- a/redbox-core/redbox/models/settings.py +++ b/redbox-core/redbox/models/settings.py @@ -10,9 +10,7 @@ log = logging.getLogger() -VANILLA_SYSTEM_PROMPT = ( - "You are an AI assistant called Redbox tasked with answering questions and providing information objectively." -) +VANILLA_SYSTEM_PROMPT = "You are an AI assistant called Redbox tasked with answering questions and providing information objectively." RETRIEVAL_SYSTEM_PROMPT = ( "Given the following conversation and extracted parts of a long document and a question, create a final answer. \n" @@ -66,15 +64,21 @@ VANILLA_QUESTION_PROMPT = "{question}\n=========\n Response: " -RETRIEVAL_QUESTION_PROMPT = "{question} \n=========\n{formatted_documents}\n=========\nFINAL ANSWER: " +RETRIEVAL_QUESTION_PROMPT = ( + "{question} \n=========\n{formatted_documents}\n=========\nFINAL ANSWER: " +) -SUMMARISATION_QUESTION_PROMPT = "Question: {question}. \n\n Documents: \n\n {documents} \n\n Answer: " +SUMMARISATION_QUESTION_PROMPT = ( + "Question: {question}. \n\n Documents: \n\n {documents} \n\n Answer: " +) MAP_QUESTION_PROMPT = "Question: {question}. " MAP_DOCUMENT_PROMPT = "\n\n Documents: \n\n {documents} \n\n Answer: " -REDUCE_QUESTION_PROMPT = "Question: {question}. \n\n Documents: \n\n {summaries} \n\n Answer: " +REDUCE_QUESTION_PROMPT = ( + "Question: {question}. \n\n Documents: \n\n {summaries} \n\n Answer: " +) CONDENSE_QUESTION_PROMPT = "{question}\n=========\n Standalone question: " @@ -136,6 +140,7 @@ class Settings(BaseSettings): anthropic_api_key: str | None = None openai_api_key: str | None = None + openai_model: str | None = None azure_openai_api_key: str | None = None azure_openai_endpoint: str | None = None @@ -183,7 +188,9 @@ class Settings(BaseSettings): dev_mode: bool = False superuser_email: str | None = None - model_config = SettingsConfigDict(env_file=".env", env_nested_delimiter="__", extra="allow", frozen=True) + model_config = SettingsConfigDict( + env_file=".env", env_nested_delimiter="__", extra="allow", frozen=True + ) def elasticsearch_client(self) -> Elasticsearch: if isinstance(self.elastic, ElasticLocalSettings): @@ -204,7 +211,9 @@ def elasticsearch_client(self) -> Elasticsearch: log.info("Cloud ID = %s", self.elastic.cloud_id) log.info("Elastic Cloud API Key = %s", self.elastic.api_key) - return Elasticsearch(cloud_id=self.elastic.cloud_id, api_key=self.elastic.api_key) + return Elasticsearch( + cloud_id=self.elastic.cloud_id, api_key=self.elastic.api_key + ) def s3_client(self): if self.object_store == "minio": From ef0993f78fcbf93cd3d41e1298f578ed24f22144 Mon Sep 17 00:00:00 2001 From: Isobel Daley Date: Thu, 4 Jul 2024 16:37:40 +0100 Subject: [PATCH 002/153] fix: remove auto-formatting changes --- core_api/src/dependencies.py | 27 ++++++++------------------- redbox-core/redbox/models/settings.py | 24 ++++++++---------------- 2 files changed, 16 insertions(+), 35 deletions(-) diff --git a/core_api/src/dependencies.py b/core_api/src/dependencies.py index d90c8594e..4c03d2837 100644 --- a/core_api/src/dependencies.py +++ b/core_api/src/dependencies.py @@ -12,16 +12,13 @@ from langchain_core.retrievers import BaseRetriever from langchain_core.runnables import ConfigurableField from langchain_elasticsearch import ApproxRetrievalStrategy, ElasticsearchStore + +from core_api.src.callbacks import LoggerCallbackHandler +from core_api.src.retriever import AllElasticsearchRetriever, ParameterisedElasticsearchRetriever from redbox.model_db import MODEL_PATH from redbox.models import Settings from redbox.storage import ElasticsearchStorageHandler -from core_api.src.callbacks import LoggerCallbackHandler -from core_api.src.retriever import ( - AllElasticsearchRetriever, - ParameterisedElasticsearchRetriever, -) - logging.basicConfig(level=logging.INFO) log = logging.getLogger() @@ -32,17 +29,13 @@ def get_env() -> Settings: @lru_cache(1) -def get_elasticsearch_client( - env: Annotated[Settings, Depends(get_env)] -) -> Elasticsearch: +def get_elasticsearch_client(env: Annotated[Settings, Depends(get_env)]) -> Elasticsearch: return env.elasticsearch_client() @lru_cache(1) def get_embedding_model(env: Annotated[Settings, Depends(get_env)]) -> Embeddings: - embedding_model = SentenceTransformerEmbeddings( - model_name=env.embedding_model, cache_folder=MODEL_PATH - ) + embedding_model = SentenceTransformerEmbeddings(model_name=env.embedding_model, cache_folder=MODEL_PATH) log.info("Loaded embedding model from environment: %s", env.embedding_model) return embedding_model @@ -79,8 +72,7 @@ def get_vector_store( @lru_cache(1) def get_parameterised_retriever( - env: Annotated[Settings, Depends(get_env)], - es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], + env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], ) -> BaseRetriever: """Creates an Elasticsearch retriever runnable. @@ -102,17 +94,14 @@ def get_parameterised_retriever( embedding_model=get_embedding_model(env), ).configurable_fields( params=ConfigurableField( - id="params", - name="Retriever parameters", - description="A dictionary of parameters to use for the retriever.", + id="params", name="Retriever parameters", description="A dictionary of parameters to use for the retriever.", ) ) @lru_cache(1) def get_all_chunks_retriever( - env: Annotated[Settings, Depends(get_env)], - es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], + env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], ): return AllElasticsearchRetriever( es_client=es, diff --git a/redbox-core/redbox/models/settings.py b/redbox-core/redbox/models/settings.py index fb1534980..f899fb77e 100644 --- a/redbox-core/redbox/models/settings.py +++ b/redbox-core/redbox/models/settings.py @@ -10,7 +10,9 @@ log = logging.getLogger() -VANILLA_SYSTEM_PROMPT = "You are an AI assistant called Redbox tasked with answering questions and providing information objectively." +VANILLA_SYSTEM_PROMPT = ( + "You are an AI assistant called Redbox tasked with answering questions and providing information objectively." +) RETRIEVAL_SYSTEM_PROMPT = ( "Given the following conversation and extracted parts of a long document and a question, create a final answer. \n" @@ -64,21 +66,15 @@ VANILLA_QUESTION_PROMPT = "{question}\n=========\n Response: " -RETRIEVAL_QUESTION_PROMPT = ( - "{question} \n=========\n{formatted_documents}\n=========\nFINAL ANSWER: " -) +RETRIEVAL_QUESTION_PROMPT = "{question} \n=========\n{formatted_documents}\n=========\nFINAL ANSWER: " -SUMMARISATION_QUESTION_PROMPT = ( - "Question: {question}. \n\n Documents: \n\n {documents} \n\n Answer: " -) +SUMMARISATION_QUESTION_PROMPT = "Question: {question}. \n\n Documents: \n\n {documents} \n\n Answer: " MAP_QUESTION_PROMPT = "Question: {question}. " MAP_DOCUMENT_PROMPT = "\n\n Documents: \n\n {documents} \n\n Answer: " -REDUCE_QUESTION_PROMPT = ( - "Question: {question}. \n\n Documents: \n\n {summaries} \n\n Answer: " -) +REDUCE_QUESTION_PROMPT = "Question: {question}. \n\n Documents: \n\n {summaries} \n\n Answer: " CONDENSE_QUESTION_PROMPT = "{question}\n=========\n Standalone question: " @@ -188,9 +184,7 @@ class Settings(BaseSettings): dev_mode: bool = False superuser_email: str | None = None - model_config = SettingsConfigDict( - env_file=".env", env_nested_delimiter="__", extra="allow", frozen=True - ) + model_config = SettingsConfigDict(env_file=".env", env_nested_delimiter="__", extra="allow", frozen=True) def elasticsearch_client(self) -> Elasticsearch: if isinstance(self.elastic, ElasticLocalSettings): @@ -211,9 +205,7 @@ def elasticsearch_client(self) -> Elasticsearch: log.info("Cloud ID = %s", self.elastic.cloud_id) log.info("Elastic Cloud API Key = %s", self.elastic.api_key) - return Elasticsearch( - cloud_id=self.elastic.cloud_id, api_key=self.elastic.api_key - ) + return Elasticsearch(cloud_id=self.elastic.cloud_id, api_key=self.elastic.api_key) def s3_client(self): if self.object_store == "minio": From 3d7ad66953c502946b79baa0f7e3c6b3039280f2 Mon Sep 17 00:00:00 2001 From: Isobel Daley Date: Thu, 4 Jul 2024 16:43:13 +0100 Subject: [PATCH 003/153] fix: remove auto-formatting changes --- core_api/src/dependencies.py | 6 +++--- redbox-core/redbox/models/settings.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core_api/src/dependencies.py b/core_api/src/dependencies.py index 4c03d2837..fd477fb77 100644 --- a/core_api/src/dependencies.py +++ b/core_api/src/dependencies.py @@ -72,7 +72,7 @@ def get_vector_store( @lru_cache(1) def get_parameterised_retriever( - env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], + env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)] ) -> BaseRetriever: """Creates an Elasticsearch retriever runnable. @@ -94,14 +94,14 @@ def get_parameterised_retriever( embedding_model=get_embedding_model(env), ).configurable_fields( params=ConfigurableField( - id="params", name="Retriever parameters", description="A dictionary of parameters to use for the retriever.", + id="params", name="Retriever parameters", description="A dictionary of parameters to use for the retriever." ) ) @lru_cache(1) def get_all_chunks_retriever( - env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], + env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)] ): return AllElasticsearchRetriever( es_client=es, diff --git a/redbox-core/redbox/models/settings.py b/redbox-core/redbox/models/settings.py index f899fb77e..b7f1d69f8 100644 --- a/redbox-core/redbox/models/settings.py +++ b/redbox-core/redbox/models/settings.py @@ -10,7 +10,7 @@ log = logging.getLogger() -VANILLA_SYSTEM_PROMPT = ( +VANILLA_SYSTEM_PROMPT = ( "You are an AI assistant called Redbox tasked with answering questions and providing information objectively." ) From 6a3f9128ccbd5936e95ffcaff7bf18e1308b0ac6 Mon Sep 17 00:00:00 2001 From: Simon Strong Date: Tue, 9 Jul 2024 13:08:18 +0100 Subject: [PATCH 004/153] SSO integration with v0.4 --- Makefile | 12 +- django_app/poetry.lock | 22 ++- django_app/pyproject.toml | 1 + .../redbox_app/redbox_core/auth_views.py | 3 + django_app/redbox_app/redbox_core/models.py | 187 ++++++++++++------ django_app/redbox_app/settings.py | 28 ++- django_app/redbox_app/urls.py | 3 + django_app/start.sh | 1 + redbox-core/redbox/models/settings.py | 8 +- 9 files changed, 191 insertions(+), 74 deletions(-) diff --git a/Makefile b/Makefile index aae2c61d8..788065385 100644 --- a/Makefile +++ b/Makefile @@ -107,15 +107,17 @@ docs-build: ## Build documentation poetry run mkdocs build # Docker +AWS_ACCOUNT_ID=REPLACE_ME + AWS_REGION=eu-west-2 -APP_NAME=redbox +APP_NAME=dbt ECR_URL=$(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com ECR_REPO_URL=$(ECR_URL)/$(ECR_REPO_NAME) -IMAGE=$(ECR_REPO_URL):$(IMAGE_TAG) +IMAGE=latest ECR_REPO_NAME=$(APP_NAME) -PREV_IMAGE_TAG=$$(git rev-parse HEAD~1) -IMAGE_TAG=$$(git rev-parse HEAD) +PREV_IMAGE_TAG=latest +IMAGE_TAG=latest tf_build_args=-var "image_tag=$(IMAGE_TAG)" DOCKER_SERVICES=$$(docker compose config --services | grep -v mlflow) @@ -152,7 +154,7 @@ docker_build: ## Build the docker container PREV_IMAGE="$(ECR_REPO_URL)-$$service:$(PREV_IMAGE_TAG)"; \ echo "Pulling previous image: $$PREV_IMAGE"; \ docker pull $$PREV_IMAGE; \ - docker compose build $$service; \ + DOCKER_DEFAULT_PLATFORM=linux/amd64 docker compose build $$service; \ else \ echo "Skipping $$service uses default image"; \ fi; \ diff --git a/django_app/poetry.lock b/django_app/poetry.lock index bd87f0d4b..28cff50d8 100644 --- a/django_app/poetry.lock +++ b/django_app/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "asgiref" @@ -1266,6 +1266,24 @@ files = [ [package.dependencies] Django = ">=3.0" +[[package]] +name = "django-staff-sso-client" +version = "4.3.0" +description = "Reusable Django app to facilitate gov.uk Staff Single Sign On" +optional = false +python-versions = "*" +files = [ + {file = "django_staff_sso_client-4.3.0-py3-none-any.whl", hash = "sha256:873279b74cc40517af6b5c6c043db74e66d3d4f147ed1495ec357ac26c14d6fa"}, + {file = "django_staff_sso_client-4.3.0.tar.gz", hash = "sha256:4f320c5c5da02a9da9f5da90b32749ff3a0ad0dcf51eb758fb85fd0e932b5261"}, +] + +[package.dependencies] +Django = ">=4.2.10,<6.0" +requests-oauthlib = "*" + +[package.extras] +test = ["build", "codecov", "flake8 (==4.0.1)", "pytest (==7.1.1)", "pytest-cov", "pytest-django", "raven", "requests-mock", "setuptools", "twine", "wheel"] + [[package]] name = "django-storages" version = "1.14.3" @@ -3505,4 +3523,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.11.2,<3.13" -content-hash = "c09c07df08240d35432e3140ff2d2700211e90bec468d157c6f33287a726f77e" +content-hash = "279ef1f6ec0fa10a985c56cec6303a8c33790873dbfe10ee11b364e08fbdb49b" diff --git a/django_app/pyproject.toml b/django_app/pyproject.toml index 577516e51..1a7ac0ff5 100644 --- a/django_app/pyproject.toml +++ b/django_app/pyproject.toml @@ -38,6 +38,7 @@ django-gov-notify = "^0.5.0" websockets = "^12.0" django-import-export = "^4.0" dataclasses-json = ">=0.6.7" +django-staff-sso-client = "^4.3.0" [tool.poetry.group.dev.dependencies] pytest = "^8.1.1" diff --git a/django_app/redbox_app/redbox_core/auth_views.py b/django_app/redbox_app/redbox_core/auth_views.py index 5c47e1af5..3938403da 100644 --- a/django_app/redbox_app/redbox_core/auth_views.py +++ b/django_app/redbox_app/redbox_core/auth_views.py @@ -5,6 +5,7 @@ from django.shortcuts import redirect, render from magic_link.models import MagicLink from requests import HTTPError +from django.conf import settings from redbox_app.redbox_core import email_handler, models from redbox_app.redbox_core.forms import SignInForm @@ -15,6 +16,8 @@ def sign_in_view(request: HttpRequest): if request.user.is_authenticated: return redirect("homepage") + if settings.LOGIN_METHOD == "sso": + return redirect("/auth/login/") if request.method == "POST": form = SignInForm(request.POST) if form.is_valid(): diff --git a/django_app/redbox_app/redbox_core/models.py b/django_app/redbox_app/redbox_core/models.py index 4495c9175..575ba8a73 100644 --- a/django_app/redbox_app/redbox_core/models.py +++ b/django_app/redbox_app/redbox_core/models.py @@ -8,10 +8,15 @@ from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ -from django_use_email_as_username.models import BaseUser, BaseUserManager from jose import jwt from yarl import URL +if settings.LOGIN_METHOD == "sso": + from django.contrib.auth.base_user import BaseUserManager + from django.contrib.auth.models import AbstractBaseUser, PermissionsMixin +else: + from django_use_email_as_username.models import BaseUser, BaseUserManager + logger = logging.getLogger(__name__) @@ -37,65 +42,55 @@ class BusinessUnit(UUIDPrimaryKeyBase): def __str__(self) -> str: # pragma: no cover return f"{self.name}" - -class User(BaseUser, UUIDPrimaryKeyBase): - class UserGrade(models.TextChoices): - AA = "AA", _("AA") - AO = "AO", _("AO") - DEPUTY_DIRECTOR = "DD", _("Deputy Director") - DIRECTOR = "D", _("Director") - DIRECTOR_GENERAL = "DG", _("Director General") - EO = "EO", _("EO") - G6 = "G6", _("G6") - G7 = "G7", _("G7") - HEO = "HEO", _("HEO") - PS = "PS", _("Permanent Secretary") - SEO = "SEO", _("SEO") - OT = "OT", _("Other") - - class Profession(models.TextChoices): - AN = "AN", _("Analysis") - CM = "CMC", _("Commercial") - COM = "COM", _("Communications") - CFIN = "CFIN", _("Corporate finance") - CF = "CF", _("Counter fraud") - DDT = "DDT", _("Digital, data and technology") - EC = "EC", _("Economics") - FIN = "FIN", _("Finance") - FEDG = "FEDG", _("Fraud, error, debts and grants") - HR = "HR", _("Human resources") - IA = "IA", _("Intelligence analysis") - IAUD = "IAUD", _("Internal audit") - IT = "IT", _("International trade") - KIM = "KIM", _("Knowledge and information management") - LG = "LG", _("Legal") - MD = "MD", _("Medical") - OP = "OP", _("Occupational psychology") - OD = "OD", _("Operational delivery") - OR = "OR", _("Operational research") - PL = "PL", _("Planning") - PI = "PI", _("Planning inspection") - POL = "POL", _("Policy") - PD = "PD", _("Project delivery") - PR = "PR", _("Property") - SE = "SE", _("Science and engineering") - SC = "SC", _("Security") - SR = "SR", _("Social research") - ST = "ST", _("Statistics") - TX = "TX", _("Tax") - VET = "VET", _("Veterinary") - OT = "OT", _("Other") - - username = None - verified = models.BooleanField(default=False, blank=True, null=True) - invited_at = models.DateTimeField(default=None, blank=True, null=True) - invite_accepted_at = models.DateTimeField(default=None, blank=True, null=True) - last_token_sent_at = models.DateTimeField(editable=False, blank=True, null=True) - password = models.CharField("password", max_length=128, blank=True, null=True) +class RedboxUserManager(BaseUserManager): + + use_in_migrations = True + + def _create_user(self, username, password, **extra_fields): + """Create and save a User with the given email and password.""" + if not username: + raise ValueError("The given email must be set") + #email = self.normalize_email(email) + User = self.model(email=email, **extra_fields) + User.set_password(password) + User.save(using=self._db) + return User + + def create_user(self, username, password=None, **extra_fields): + """Create and save a regular User with the given email and password.""" + extra_fields.setdefault("is_staff", False) + extra_fields.setdefault("is_superuser", False) + return self._create_user(username, password, **extra_fields) + + def create_superuser(self, username, password, **extra_fields): + """Create and save a SuperUser with the given email and password.""" + extra_fields.setdefault("is_staff", True) + extra_fields.setdefault("is_superuser", True) + + if extra_fields.get("is_staff") is not True: + raise ValueError("Superuser must have is_staff=True.") + if extra_fields.get("is_superuser") is not True: + raise ValueError("Superuser must have is_superuser=True.") + + return self._create_user(username, password, **extra_fields) + +class User(AbstractBaseUser, PermissionsMixin, UUIDPrimaryKeyBase): + username = models.EmailField(unique=True, default="default@default.co.uk") + password = models.CharField(default="fakepassword") + email = models.EmailField(unique=True) + first_name = models.CharField(max_length=48) + last_name = models.CharField(max_length=48) + is_staff = models.BooleanField(default=False) + is_active = models.BooleanField(default=True) + is_superuser = models.BooleanField(default=False) + date_joined = models.DateTimeField(default=timezone.now) business_unit = models.ForeignKey(BusinessUnit, null=True, blank=True, on_delete=models.SET_NULL) - grade = models.CharField(null=True, blank=True, max_length=3, choices=UserGrade) - profession = models.CharField(null=True, blank=True, max_length=4, choices=Profession) - objects = BaseUserManager() + grade = models.CharField(null=True, blank=True, max_length=3) + profession = models.CharField(null=True, blank=True, max_length=4) + + USERNAME_FIELD = 'username' + REQUIRED_FIELDS = [] + objects = RedboxUserManager() def __str__(self) -> str: # pragma: no cover return f"{self.email}" @@ -108,7 +103,79 @@ def get_bearer_token(self) -> str: """the bearer token expected by the core-api""" user_uuid = str(self.id) bearer_token = jwt.encode({"user_uuid": user_uuid}, key=settings.SECRET_KEY) - return f"Bearer {bearer_token}" + return f"Bearer {bearer_token}" + +# class User(BaseUser, UUIDPrimaryKeyBase): +# class UserGrade(models.TextChoices): +# AA = "AA", _("AA") +# AO = "AO", _("AO") +# DEPUTY_DIRECTOR = "DD", _("Deputy Director") +# DIRECTOR = "D", _("Director") +# DIRECTOR_GENERAL = "DG", _("Director General") +# EO = "EO", _("EO") +# G6 = "G6", _("G6") +# G7 = "G7", _("G7") +# HEO = "HEO", _("HEO") +# PS = "PS", _("Permanent Secretary") +# SEO = "SEO", _("SEO") +# OT = "OT", _("Other") + +# class Profession(models.TextChoices): +# AN = "AN", _("Analysis") +# CM = "CMC", _("Commercial") +# COM = "COM", _("Communications") +# CFIN = "CFIN", _("Corporate finance") +# CF = "CF", _("Counter fraud") +# DDT = "DDT", _("Digital, data and technology") +# EC = "EC", _("Economics") +# FIN = "FIN", _("Finance") +# FEDG = "FEDG", _("Fraud, error, debts and grants") +# HR = "HR", _("Human resources") +# IA = "IA", _("Intelligence analysis") +# IAUD = "IAUD", _("Internal audit") +# IT = "IT", _("International trade") +# KIM = "KIM", _("Knowledge and information management") +# LG = "LG", _("Legal") +# MD = "MD", _("Medical") +# OP = "OP", _("Occupational psychology") +# OD = "OD", _("Operational delivery") +# OR = "OR", _("Operational research") +# PL = "PL", _("Planning") +# PI = "PI", _("Planning inspection") +# POL = "POL", _("Policy") +# PD = "PD", _("Project delivery") +# PR = "PR", _("Property") +# SE = "SE", _("Science and engineering") +# SC = "SC", _("Security") +# SR = "SR", _("Social research") +# ST = "ST", _("Statistics") +# TX = "TX", _("Tax") +# VET = "VET", _("Veterinary") +# OT = "OT", _("Other") + +# username = None +# verified = models.BooleanField(default=False, blank=True, null=True) +# invited_at = models.DateTimeField(default=None, blank=True, null=True) +# invite_accepted_at = models.DateTimeField(default=None, blank=True, null=True) +# last_token_sent_at = models.DateTimeField(editable=False, blank=True, null=True) +# password = models.CharField("password", max_length=128, blank=True, null=True) +# business_unit = models.ForeignKey(BusinessUnit, null=True, blank=True, on_delete=models.SET_NULL) +# grade = models.CharField(null=True, blank=True, max_length=3, choices=UserGrade) +# profession = models.CharField(null=True, blank=True, max_length=4, choices=Profession) +# objects = BaseUserManager() + +# def __str__(self) -> str: # pragma: no cover +# return f"{self.email}" + +# def save(self, *args, **kwargs): +# self.email = self.email.lower() +# super().save(*args, **kwargs) + +# def get_bearer_token(self) -> str: +# """the bearer token expected by the core-api""" +# user_uuid = str(self.id) +# bearer_token = jwt.encode({"user_uuid": user_uuid}, key=settings.SECRET_KEY) +# return f"Bearer {bearer_token}" class StatusEnum(models.TextChoices): diff --git a/django_app/redbox_app/settings.py b/django_app/redbox_app/settings.py index b68ed4b36..2daf118c3 100644 --- a/django_app/redbox_app/settings.py +++ b/django_app/redbox_app/settings.py @@ -4,6 +4,7 @@ from pathlib import Path import environ +from django.urls import reverse_lazy import sentry_sdk from dotenv import load_dotenv from import_export.formats.base_formats import CSV @@ -19,6 +20,8 @@ SECRET_KEY = env.str("DJANGO_SECRET_KEY") ENVIRONMENT = Environment[env.str("ENVIRONMENT").upper()] WEBSOCKET_SCHEME = "ws" if ENVIRONMENT.is_test else "wss" +LOGIN_METHOD = env.str("LOGIN_METHOD") + # SECURITY WARNING: don't run with debug turned on in production! DEBUG = env.bool("DEBUG") @@ -65,6 +68,9 @@ "import_export", ] +if LOGIN_METHOD == "sso": + INSTALLED_APPS.append("authbroker_client") + MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", "whitenoise.middleware.WhiteNoiseMiddleware", @@ -117,6 +123,9 @@ "django.contrib.auth.backends.ModelBackend", ] +if LOGIN_METHOD == "sso": + AUTHENTICATION_BACKENDS.append("authbroker_client.backends.AuthbrokerBackend") + AUTH_PASSWORD_VALIDATORS = [ { "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", @@ -144,8 +153,8 @@ SITE_ID = 1 AUTH_USER_MODEL = "redbox_core.User" ACCOUNT_EMAIL_VERIFICATION = "none" -LOGIN_REDIRECT_URL = "homepage" -LOGIN_URL = "sign-in" +# LOGIN_REDIRECT_URL = "homepage" +# LOGIN_URL = "sign-in" # CSP settings https://content-security-policy.com/ # https://django-csp.readthedocs.io/ @@ -232,7 +241,7 @@ ALLOWED_HOSTS = ENVIRONMENT.hosts else: LOCALHOST = socket.gethostbyname(socket.gethostname()) - ALLOWED_HOSTS = [LOCALHOST, *ENVIRONMENT.hosts] + ALLOWED_HOSTS = [LOCALHOST, *ENVIRONMENT.hosts,"redbox-sandbox.uktrade.digital","dbt-default-alb-1550180991.eu-west-2.elb.amazonaws.com"] if not ENVIRONMENT.is_local: SENTRY_DSN = env.str("SENTRY_DSN", None) @@ -331,3 +340,16 @@ FILE_EXPIRY_IN_SECONDS = env.int("FILE_EXPIRY_IN_DAYS") * 24 * 60 * 60 SUPERUSER_EMAIL = env.str("SUPERUSER_EMAIL", None) MAX_SECURITY_CLASSIFICATION = Classification[env.str("MAX_SECURITY_CLASSIFICATION")] + +if LOGIN_METHOD == "sso": + AUTHBROKER_URL = env.str("AUTHBROKER_URL") + AUTHBROKER_CLIENT_ID = env.str("AUTHBROKER_CLIENT_ID") + AUTHBROKER_CLIENT_SECRET = env.str("AUTHBROKER_CLIENT_SECRET") + LOGIN_URL = reverse_lazy("authbroker_client:login") + LOGIN_REDIRECT_URL = reverse_lazy("homepage") +elif LOGIN_METHOD == "magic_link": + LOGIN_REDIRECT_URL = "homepage" + LOGIN_URL = "sign-in" +else: + LOGIN_REDIRECT_URL = "homepage" + LOGIN_URL = "sign-in" \ No newline at end of file diff --git a/django_app/redbox_app/urls.py b/django_app/redbox_app/urls.py index 848c9c0e9..5ff06a671 100644 --- a/django_app/redbox_app/urls.py +++ b/django_app/redbox_app/urls.py @@ -17,6 +17,9 @@ path("signed-out/", auth_views.signed_out_view, name="signed-out"), ] +if settings.LOGIN_METHOD == "sso": + auth_urlpatterns.append(path("auth/", include("authbroker_client.urls"))) + info_urlpatterns = [ path("privacy-notice/", info_views.privacy_notice_view, name="privacy-notice"), path( diff --git a/django_app/start.sh b/django_app/start.sh index b4219c68e..4c29439cd 100644 --- a/django_app/start.sh +++ b/django_app/start.sh @@ -1,5 +1,6 @@ #!/bin/sh +venv/bin/django-admin makemigrations venv/bin/django-admin migrate venv/bin/django-admin collectstatic --noinput venv/bin/django-admin compress --force --engine jinja2 diff --git a/redbox-core/redbox/models/settings.py b/redbox-core/redbox/models/settings.py index 2716eba79..8ad2416b7 100644 --- a/redbox-core/redbox/models/settings.py +++ b/redbox-core/redbox/models/settings.py @@ -110,12 +110,12 @@ class ElasticLocalSettings(BaseModel): model_config = SettingsConfigDict(frozen=True) - host: str = "elasticsearch" + host: str = "elasticsearch.internal" port: int = 9200 scheme: str = "http" user: str = "elastic" - version: str = "8.11.0" - password: str = "redboxpass" + version: str = "8.14.0" + password: str = "IngKKFLDMtGRF5Xb1L2Z" subscription_level: str = "basic" @@ -150,7 +150,7 @@ class Settings(BaseSettings): partition_strategy: Literal["auto", "fast", "ocr_only", "hi_res"] = "fast" clustering_strategy: Literal["full"] | None = None - elastic: ElasticCloudSettings | ElasticLocalSettings = ElasticLocalSettings() + elastic: ElasticLocalSettings = ElasticLocalSettings() elastic_root_index: str = "redbox-data" kibana_system_password: str = "redboxpass" From ed4a0e4dca9fd87c3642bf7c6a1e25a6107bf473 Mon Sep 17 00:00:00 2001 From: Isobel Daley Date: Tue, 9 Jul 2024 14:28:01 +0100 Subject: [PATCH 005/153] feat:make elastic host and password configurable --- redbox-core/redbox/models/settings.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/redbox-core/redbox/models/settings.py b/redbox-core/redbox/models/settings.py index 8ad2416b7..53c4b1df3 100644 --- a/redbox-core/redbox/models/settings.py +++ b/redbox-core/redbox/models/settings.py @@ -110,13 +110,13 @@ class ElasticLocalSettings(BaseModel): model_config = SettingsConfigDict(frozen=True) - host: str = "elasticsearch.internal" - port: int = 9200 - scheme: str = "http" - user: str = "elastic" - version: str = "8.14.0" - password: str = "IngKKFLDMtGRF5Xb1L2Z" - subscription_level: str = "basic" + elastic_host: str | None = None + elastic_port: int = 9200 + elastic_scheme: str = "http" + elastic_user: str = "elastic" + elastic_version: str = "8.14.0" + elastic_password: str | None = None + elastic_subscription_level: str = "basic" class ElasticCloudSettings(BaseModel): @@ -192,12 +192,12 @@ def elasticsearch_client(self) -> Elasticsearch: return Elasticsearch( hosts=[ { - "host": self.elastic.host, - "port": self.elastic.port, - "scheme": self.elastic.scheme, + "host": self.elastic.elastic_host, + "port": self.elastic.elastic_port, + "scheme": self.elastic.elastic_scheme, } ], - basic_auth=(self.elastic.user, self.elastic.password), + basic_auth=(self.elastic.elastic_user, self.elastic.elastic_password), ) log.info("Connecting to Elastic Cloud Cluster") From 05e7afa882b7055bb2e460bf56ad7441eb3e73a7 Mon Sep 17 00:00:00 2001 From: Isobel Daley Date: Tue, 9 Jul 2024 15:47:04 +0100 Subject: [PATCH 006/153] fix: host updated to elastic_host --- redbox-core/redbox/models/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redbox-core/redbox/models/settings.py b/redbox-core/redbox/models/settings.py index 53c4b1df3..7114df747 100644 --- a/redbox-core/redbox/models/settings.py +++ b/redbox-core/redbox/models/settings.py @@ -188,7 +188,7 @@ class Settings(BaseSettings): def elasticsearch_client(self) -> Elasticsearch: if isinstance(self.elastic, ElasticLocalSettings): log.info("Connecting to self managed Elasticsearch") - log.info("Elasticsearch host = %s", self.elastic.host) + log.info("Elasticsearch host = %s", self.elastic.elastic_host) return Elasticsearch( hosts=[ { From 1a170bfb5ff872ddf5158f11f1be8d06016894ae Mon Sep 17 00:00:00 2001 From: Isobel Daley Date: Tue, 9 Jul 2024 16:22:39 +0100 Subject: [PATCH 007/153] fix: update elastic_host and elastic_password --- redbox-core/redbox/models/settings.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/redbox-core/redbox/models/settings.py b/redbox-core/redbox/models/settings.py index 7114df747..90a19fe32 100644 --- a/redbox-core/redbox/models/settings.py +++ b/redbox-core/redbox/models/settings.py @@ -5,6 +5,11 @@ from elasticsearch import Elasticsearch from pydantic import BaseModel from pydantic_settings import BaseSettings, SettingsConfigDict +import environ + +load_dotenv() + +env = environ.Env() logging.basicConfig(level=logging.INFO) log = logging.getLogger() @@ -110,12 +115,12 @@ class ElasticLocalSettings(BaseModel): model_config = SettingsConfigDict(frozen=True) - elastic_host: str | None = None + elastic_host: str = env.str("ELASTIC_HOST") elastic_port: int = 9200 elastic_scheme: str = "http" elastic_user: str = "elastic" elastic_version: str = "8.14.0" - elastic_password: str | None = None + elastic_password: str = env.str("ELASTIC__PASSWORD") elastic_subscription_level: str = "basic" From 43fb6d3c04dc5f9dc9209df6d0a2ca649719e89c Mon Sep 17 00:00:00 2001 From: Isobel Daley Date: Tue, 9 Jul 2024 16:37:28 +0100 Subject: [PATCH 008/153] fix: remove default values in env files --- .env.django | 4 ++-- .env.example | 4 ++-- .env.integration | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env.django b/.env.django index 8aa7ddaed..c8916443c 100644 --- a/.env.django +++ b/.env.django @@ -10,10 +10,10 @@ DEV_MODE=true # === Database === -ELASTIC__HOST=elasticsearch +ELASTIC__HOST= ELASTIC__VERSION=8.11.0 ELASTIC__USER=elastic -ELASTIC__PASSWORD=redboxpass +ELASTIC__PASSWORD= ELASTIC__PORT=9200 ELASTIC__SCHEME=http diff --git a/.env.example b/.env.example index 9a78c01e5..2eed6fe79 100644 --- a/.env.example +++ b/.env.example @@ -10,10 +10,10 @@ DEV_MODE=true # === Database === -ELASTIC__HOST=elasticsearch +ELASTIC__HOST= ELASTIC__VERSION=8.11.0 ELASTIC__USER=elastic -ELASTIC__PASSWORD=redboxpass +ELASTIC__PASSWORD= ELASTIC__PORT=9200 ELASTIC__SCHEME=http ELASTIC__SUBSCRIPTION_LEVEL=basic diff --git a/.env.integration b/.env.integration index e3c07b55d..9eb69b9d0 100644 --- a/.env.integration +++ b/.env.integration @@ -5,10 +5,10 @@ LLM_MAX_TOKENS=1024 # === Database === -ELASTIC__HOST=elasticsearch +ELASTIC__HOST= ELASTIC__VERSION=8.11.0 ELASTIC__USER=elastic -ELASTIC__PASSWORD=redboxpass +ELASTIC__PASSWORD= ELASTIC__PORT=9200 ELASTIC__SCHEME=http ELASTIC__SUBSCRIPTION_LEVEL=basic From ad702b1ef8b7c539ae7930703e521d77bf6f20fe Mon Sep 17 00:00:00 2001 From: Isobel Daley Date: Tue, 9 Jul 2024 22:27:30 +0100 Subject: [PATCH 009/153] fix: revert environment variable changes --- .env.django | 4 ++-- .env.example | 4 ++-- .env.integration | 4 ++-- redbox-core/redbox/models/settings.py | 29 +++++++++++---------------- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/.env.django b/.env.django index c8916443c..8aa7ddaed 100644 --- a/.env.django +++ b/.env.django @@ -10,10 +10,10 @@ DEV_MODE=true # === Database === -ELASTIC__HOST= +ELASTIC__HOST=elasticsearch ELASTIC__VERSION=8.11.0 ELASTIC__USER=elastic -ELASTIC__PASSWORD= +ELASTIC__PASSWORD=redboxpass ELASTIC__PORT=9200 ELASTIC__SCHEME=http diff --git a/.env.example b/.env.example index 2eed6fe79..9a78c01e5 100644 --- a/.env.example +++ b/.env.example @@ -10,10 +10,10 @@ DEV_MODE=true # === Database === -ELASTIC__HOST= +ELASTIC__HOST=elasticsearch ELASTIC__VERSION=8.11.0 ELASTIC__USER=elastic -ELASTIC__PASSWORD= +ELASTIC__PASSWORD=redboxpass ELASTIC__PORT=9200 ELASTIC__SCHEME=http ELASTIC__SUBSCRIPTION_LEVEL=basic diff --git a/.env.integration b/.env.integration index 9eb69b9d0..e3c07b55d 100644 --- a/.env.integration +++ b/.env.integration @@ -5,10 +5,10 @@ LLM_MAX_TOKENS=1024 # === Database === -ELASTIC__HOST= +ELASTIC__HOST=elasticsearch ELASTIC__VERSION=8.11.0 ELASTIC__USER=elastic -ELASTIC__PASSWORD= +ELASTIC__PASSWORD=redboxpass ELASTIC__PORT=9200 ELASTIC__SCHEME=http ELASTIC__SUBSCRIPTION_LEVEL=basic diff --git a/redbox-core/redbox/models/settings.py b/redbox-core/redbox/models/settings.py index 90a19fe32..aa5f31610 100644 --- a/redbox-core/redbox/models/settings.py +++ b/redbox-core/redbox/models/settings.py @@ -5,11 +5,7 @@ from elasticsearch import Elasticsearch from pydantic import BaseModel from pydantic_settings import BaseSettings, SettingsConfigDict -import environ -load_dotenv() - -env = environ.Env() logging.basicConfig(level=logging.INFO) log = logging.getLogger() @@ -109,19 +105,18 @@ class AISettings(BaseModel): reduce_system_prompt: str = REDUCE_SYSTEM_PROMPT reduce_question_prompt: str = REDUCE_QUESTION_PROMPT - class ElasticLocalSettings(BaseModel): """settings required for a local/ec2 instance of elastic""" model_config = SettingsConfigDict(frozen=True) - elastic_host: str = env.str("ELASTIC_HOST") - elastic_port: int = 9200 - elastic_scheme: str = "http" - elastic_user: str = "elastic" - elastic_version: str = "8.14.0" - elastic_password: str = env.str("ELASTIC__PASSWORD") - elastic_subscription_level: str = "basic" + host: str = "elasticsearch" + port: int = 9200 + scheme: str = "http" + user: str = "elastic" + version: str = "8.11.0" + password: str = "redboxpass" + subscription_level: str = "basic" class ElasticCloudSettings(BaseModel): @@ -193,16 +188,16 @@ class Settings(BaseSettings): def elasticsearch_client(self) -> Elasticsearch: if isinstance(self.elastic, ElasticLocalSettings): log.info("Connecting to self managed Elasticsearch") - log.info("Elasticsearch host = %s", self.elastic.elastic_host) + log.info("Elasticsearch host = %s", self.elastic.host) return Elasticsearch( hosts=[ { - "host": self.elastic.elastic_host, - "port": self.elastic.elastic_port, - "scheme": self.elastic.elastic_scheme, + "host": self.elastic.host, + "port": self.elastic.port, + "scheme": self.elastic.scheme, } ], - basic_auth=(self.elastic.elastic_user, self.elastic.elastic_password), + basic_auth=(self.elastic.user, self.elastic.password), ) log.info("Connecting to Elastic Cloud Cluster") From bffaf951adf092eae812f339f0d18a6f9c706cb9 Mon Sep 17 00:00:00 2001 From: Anthoni Gleeson Date: Wed, 10 Jul 2024 10:11:19 +0100 Subject: [PATCH 010/153] add hosts env variable --- django_app/redbox_app/settings.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/django_app/redbox_app/settings.py b/django_app/redbox_app/settings.py index 2daf118c3..3606c5df7 100644 --- a/django_app/redbox_app/settings.py +++ b/django_app/redbox_app/settings.py @@ -4,15 +4,14 @@ from pathlib import Path import environ -from django.urls import reverse_lazy import sentry_sdk +from django.urls import reverse_lazy from dotenv import load_dotenv from import_export.formats.base_formats import CSV +from redbox_app.setting_enums import Classification, Environment from sentry_sdk.integrations.django import DjangoIntegration from storages.backends import s3boto3 -from redbox_app.setting_enums import Classification, Environment - load_dotenv() env = environ.Env() @@ -22,6 +21,11 @@ WEBSOCKET_SCHEME = "ws" if ENVIRONMENT.is_test else "wss" LOGIN_METHOD = env.str("LOGIN_METHOD") +if env.str("HOSTS"): + env_hosts = env.str("HOSTS", "").split(",") +else: + env_hosts = ENVIRONMENT.hosts + # SECURITY WARNING: don't run with debug turned on in production! DEBUG = env.bool("DEBUG") @@ -153,8 +157,6 @@ SITE_ID = 1 AUTH_USER_MODEL = "redbox_core.User" ACCOUNT_EMAIL_VERIFICATION = "none" -# LOGIN_REDIRECT_URL = "homepage" -# LOGIN_URL = "sign-in" # CSP settings https://content-security-policy.com/ # https://django-csp.readthedocs.io/ @@ -176,7 +178,7 @@ ) CSP_STYLE_SRC = ("'self'",) CSP_FRAME_ANCESTORS = ("'none'",) -CSP_CONNECT_SRC = ["'self'", f"wss://{ENVIRONMENT.hosts[0]}/ws/chat/", "plausible.io"] +CSP_CONNECT_SRC = ["'self'", f"wss://{env_hosts[0]}/ws/chat/", "plausible.io"] # https://pypi.org/project/django-permissions-policy/ PERMISSIONS_POLICY: dict[str, list] = { @@ -238,10 +240,13 @@ SESSION_COOKIE_SECURE = True if ENVIRONMENT.is_test: - ALLOWED_HOSTS = ENVIRONMENT.hosts + ALLOWED_HOSTS = env_hosts else: LOCALHOST = socket.gethostbyname(socket.gethostname()) - ALLOWED_HOSTS = [LOCALHOST, *ENVIRONMENT.hosts,"redbox-sandbox.uktrade.digital","dbt-default-alb-1550180991.eu-west-2.elb.amazonaws.com"] + ALLOWED_HOSTS = [ + LOCALHOST, + *env_hosts, + ] if not ENVIRONMENT.is_local: SENTRY_DSN = env.str("SENTRY_DSN", None) @@ -273,7 +278,9 @@ LOGGING = { "version": 1, "disable_existing_loggers": False, - "formatters": {"verbose": {"format": "%(asctime)s %(levelname)s %(module)s: %(message)s"}}, + "formatters": { + "verbose": {"format": "%(asctime)s %(levelname)s %(module)s: %(message)s"} + }, "handlers": { "file": { "level": LOG_LEVEL, @@ -314,7 +321,9 @@ elif EMAIL_BACKEND_TYPE == "GOVUKNOTIFY": EMAIL_BACKEND = "django_gov_notify.backends.NotifyEmailBackend" GOVUK_NOTIFY_API_KEY = env.str("GOVUK_NOTIFY_API_KEY") - GOVUK_NOTIFY_PLAIN_EMAIL_TEMPLATE_ID = env.str("GOVUK_NOTIFY_PLAIN_EMAIL_TEMPLATE_ID") + GOVUK_NOTIFY_PLAIN_EMAIL_TEMPLATE_ID = env.str( + "GOVUK_NOTIFY_PLAIN_EMAIL_TEMPLATE_ID" + ) else: message = f"Unknown EMAIL_BACKEND_TYPE of {EMAIL_BACKEND_TYPE}" raise ValueError(message) @@ -352,4 +361,4 @@ LOGIN_URL = "sign-in" else: LOGIN_REDIRECT_URL = "homepage" - LOGIN_URL = "sign-in" \ No newline at end of file + LOGIN_URL = "sign-in" From 63e872410cafc463505f9b4d8a6e58739bfb3734 Mon Sep 17 00:00:00 2001 From: Simon Strong Date: Tue, 9 Jul 2024 20:53:17 +0100 Subject: [PATCH 011/153] hot fix deployment Prod fix: rebase against dev to incorporate hosts --- django_app/redbox_app/settings.py | 1 + redbox-core/redbox/models/settings.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/django_app/redbox_app/settings.py b/django_app/redbox_app/settings.py index 3606c5df7..4e753bf09 100644 --- a/django_app/redbox_app/settings.py +++ b/django_app/redbox_app/settings.py @@ -243,6 +243,7 @@ ALLOWED_HOSTS = env_hosts else: LOCALHOST = socket.gethostbyname(socket.gethostname()) + ALLOWED_HOSTS = [LOCALHOST, *ENVIRONMENT.hosts,"redbox-trial.uktrade.digital","dbt-default-alb-1021417632.eu-west-2.elb.amazonaws.com"] ALLOWED_HOSTS = [ LOCALHOST, *env_hosts, diff --git a/redbox-core/redbox/models/settings.py b/redbox-core/redbox/models/settings.py index aa5f31610..ab4c0c505 100644 --- a/redbox-core/redbox/models/settings.py +++ b/redbox-core/redbox/models/settings.py @@ -6,7 +6,6 @@ from pydantic import BaseModel from pydantic_settings import BaseSettings, SettingsConfigDict - logging.basicConfig(level=logging.INFO) log = logging.getLogger() From be7401578ae383afdd24cfe5a31bc41dbbc786fd Mon Sep 17 00:00:00 2001 From: Isobel Daley Date: Wed, 10 Jul 2024 12:42:25 +0100 Subject: [PATCH 012/153] feat: modify HTML to reflect DBT-specific content --- .../templates/accessibility-statement.html | 71 +++++------ django_app/redbox_app/templates/base.html | 4 +- .../redbox_app/templates/documents.html | 6 + django_app/redbox_app/templates/homepage.html | 4 +- .../redbox_app/templates/privacy-notice.html | 117 +++++++++--------- django_app/redbox_app/templates/support.html | 76 ++++++------ 6 files changed, 138 insertions(+), 140 deletions(-) diff --git a/django_app/redbox_app/templates/accessibility-statement.html b/django_app/redbox_app/templates/accessibility-statement.html index 2512a1dc8..f7594f893 100644 --- a/django_app/redbox_app/templates/accessibility-statement.html +++ b/django_app/redbox_app/templates/accessibility-statement.html @@ -4,43 +4,40 @@ {% block content %} -
-
+
+

Accessibility statement for Redbox Copilot

+ +

This accessibility statement applies to Redbox Copilot.

+

This website is run by the Department for Business and Trade. Whilst the website is currently only being trialed, overtime we may want as many people as possible to be able to use this website. For example, that means you should be able to:

+
    +
  • change colours, contrast levels and fonts
  • +
  • zoom in up to 300% without the text spilling off the screen
  • +
  • navigate most of the website using just a keyboard
  • +
  • navigate most of the website using speech recognition software
  • +
  • listen to most of the website using a screen reader (including the most recent versions of JAWS, NVDA and VoiceOver)
  • +
+

We've also made the website text as simple as possible to understand.

+

AbilityNet has advice on making your device easier to use if you have a disability.

+ +

Feedback and contact information

+

If you need information on this website in a different format like accessible PDF, large print, easy read, audio recording or braille, email {{contact_email}}

+

We'll consider your request and get back to you in 14 days.

+ +

Reporting accessibility problems with this website

+

We're always looking to improve the accessibility of this website. If you find any problems not listed on this page or think we're not meeting accessibility requirements, contact the team at {{contact_email}}.

+ +

Enforcement procedure

+

The Equality and Human Rights Commission (EHRC) is responsible for enforcing the Public Sector Bodies (Websites and Mobile Applications) (No. 2) Accessibility Regulations 2018 (the "accessibility regulations"). If you're not happy with how we respond to your complaint, contact the Equality Advisory and Support Service (EASS).

+ +

Technical information about this website's accessibility

+

The Department for Business and Trade is committed to making its website accessible, in accordance with the Public Sector Bodies (Websites and Mobile Applications) (No. 2) Accessibility Regulations 2018.

+ +

Compliance status

+

This website is fully compliant with the Web Content Accessibility Guidelines version 2.2 AA standard.

+ +

Preparation of this accessibility statement

+

This statement was prepared on 27th February 2024. It was last reviewed on 21st June 2024.

-

Accessibility statement for Redbox

- -

This accessibility statement applies to Redbox.

-

This website is run by the Cabinet Office. We want as many people as possible to be able to use this website. For example, that means you should be able to:

-
    -
  • change colours, contrast levels and fonts
  • -
  • zoom in up to 300% without the text spilling off the screen
  • -
  • navigate most of the website using just a keyboard
  • -
  • navigate most of the website using speech recognition software
  • -
  • listen to most of the website using a screen reader (including the most recent versions of JAWS, NVDA and VoiceOver)
  • -
-

We've also made the website text as simple as possible to understand.

-

AbilityNet has advice on making your device easier to use if you have a disability.

- -

Feedback and contact information

-

If you need information on this website in a different format like accessible PDF, large print, easy read, audio recording or braille, email {{contact_email}}

-

We'll consider your request and get back to you in 14 days.

- -

Reporting accessibility problems with this website

-

We're always looking to improve the accessibility of this website. If you find any problems not listed on this page or think we're not meeting accessibility requirements, contact the team at {{contact_email}}.

- -

Enforcement procedure

-

The Equality and Human Rights Commission (EHRC) is responsible for enforcing the Public Sector Bodies (Websites and Mobile Applications) (No. 2) Accessibility Regulations 2018 (the "accessibility regulations"). If you're not happy with how we respond to your complaint, contact the Equality Advisory and Support Service (EASS).

- -

Technical information about this website's accessibility

-

Cabinet Office is committed to making its website accessible, in accordance with the Public Sector Bodies (Websites and Mobile Applications) (No. 2) Accessibility Regulations 2018.

- -

Compliance status

-

This website is fully compliant with the Web Content Accessibility Guidelines version 2.2 AA standard.

- -

Preparation of this accessibility statement

-

This statement was prepared on 27th February 2024. It was last reviewed on 12th April 2024.

- -
-{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/django_app/redbox_app/templates/base.html b/django_app/redbox_app/templates/base.html index 87f08bcfe..a76e763b2 100644 --- a/django_app/redbox_app/templates/base.html +++ b/django_app/redbox_app/templates/base.html @@ -57,7 +57,7 @@ {% set menu_items = [ {"text": "Documents", "href": url('documents')}, {"text": "Chats", "href": url('chats')}, - {"text": "My details", "href": url('demographics')}, + {"text": "Sign out", "href": url('signed-out')} ] %} {% else %} @@ -115,7 +115,7 @@ - +
diff --git a/django_app/redbox_app/templates/documents.html b/django_app/redbox_app/templates/documents.html index a4eb97601..5ccfcfc68 100644 --- a/django_app/redbox_app/templates/documents.html +++ b/django_app/redbox_app/templates/documents.html @@ -40,6 +40,12 @@

Your documents

Manage documents to use with your Redbox.

Redbox generates a response using only the documents you upload. This is different to other Large Language Models (LLM) such as ChatGPT and Claude.

+

Redbox can take documents you upload and use them in tandem with the AI to perform a number of functions:

+
    +
  • chat directly with a document through asking questions
  • +
  • summarise any document
  • +
+

Use @chat when sending a message to talk directly to the AI, otherwise Redbox will default to interacting with your documents.

{#
diff --git a/django_app/redbox_app/templates/homepage.html b/django_app/redbox_app/templates/homepage.html index df071c32d..4cf3899be 100644 --- a/django_app/redbox_app/templates/homepage.html +++ b/django_app/redbox_app/templates/homepage.html @@ -8,8 +8,8 @@
-

Ask any question of documents in your Redbox

-

Use Artificial Intelligence (AI) to get insights from your personal document set. You can use up to, and including, {{ security }} documents.

+

Use Artificial Intelligence (AI) to interact with Official level documents from your own document set.

+

You may summarise documents, ask questions related to one or more documents, and chat directly with the AI to assist you in your work. You can use up to, and including, Official documents.