From 4e0e25e24ef756035defabb2d11117ade40d2375 Mon Sep 17 00:00:00 2001
From: Anupam Kumar <kyteinsky@gmail.com>
Date: Wed, 30 Oct 2024 15:14:49 +0530
Subject: [PATCH 1/4] install postgresql in the docker container

Signed-off-by: Anupam Kumar <kyteinsky@gmail.com>
---
 Dockerfile                                | 41 +++++++++++-------
 config.cpu.yaml                           |  6 +--
 config.gpu.yaml                           |  6 +--
 context_chat_backend/vectordb/pgvector.py | 20 +++++++--
 dockerfile_scripts/entrypoint.sh          |  9 ++++
 dockerfile_scripts/install_deps.sh        |  4 ++
 dockerfile_scripts/install_py11.sh        |  8 ++++
 dockerfile_scripts/pgsql/env              |  7 +++
 dockerfile_scripts/pgsql/install.sh       | 24 ++++++++++
 dockerfile_scripts/pgsql/setup.sh         | 53 +++++++++++++++++++++++
 requirements.in.txt                       |  1 +
 requirements.txt                          | 33 ++++++++------
 12 files changed, 173 insertions(+), 39 deletions(-)
 create mode 100755 dockerfile_scripts/entrypoint.sh
 create mode 100755 dockerfile_scripts/install_deps.sh
 create mode 100755 dockerfile_scripts/install_py11.sh
 create mode 100644 dockerfile_scripts/pgsql/env
 create mode 100755 dockerfile_scripts/pgsql/install.sh
 create mode 100755 dockerfile_scripts/pgsql/setup.sh

diff --git a/Dockerfile b/Dockerfile
index 19653c6..2850a23 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,19 +1,33 @@
-FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04
+FROM nvidia/cuda:12.2.2-runtime-ubuntu22.04
 
-ENV DEBIAN_FRONTEND=noninteractive
+ARG CCB_DB_NAME=ccb
+ARG CCB_DB_USER=ccbuser
+ARG CCB_DB_PASS=ccbpass
 
-RUN apt-get update
-RUN apt-get install -y software-properties-common
-RUN add-apt-repository -y ppa:deadsnakes/ppa
-RUN apt-get update
-RUN apt-get install -y --no-install-recommends python3.11 python3.11-venv python3-pip vim git pciutils
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
-RUN apt-get -y clean
-RUN rm -rf /var/lib/apt/lists/*
+ENV CCB_DB_NAME ${CCB_DB_NAME}
+ENV CCB_DB_USER ${CCB_DB_USER}
+ENV CCB_DB_PASS ${CCB_DB_PASS}
+ENV DEBIAN_FRONTEND noninteractive
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute
+ENV AA_DOCKER_ENV 1
 
 # Set working directory
 WORKDIR /app
 
+# Install dependencies
+ADD dockerfile_scripts/install_deps.sh dockerfile_scripts/install_deps.sh
+RUN ./dockerfile_scripts/install_deps.sh
+ADD dockerfile_scripts/install_py11.sh dockerfile_scripts/install_py11.sh
+RUN ./dockerfile_scripts/install_py11.sh
+ADD dockerfile_scripts/pgsql dockerfile_scripts/pgsql
+RUN ./dockerfile_scripts/pgsql/install.sh
+RUN apt-get autoclean
+ADD dockerfile_scripts/entrypoint.sh dockerfile_scripts/entrypoint.sh
+
+# Restore interactivity
+ENV DEBIAN_FRONTEND dialog
+
 # Copy requirements files
 COPY requirements.txt .
 
@@ -23,15 +37,10 @@ RUN python3 -m pip install --no-cache-dir https://github.com/abetlen/llama-cpp-p
 RUN sed -i '/llama_cpp_python/d' requirements.txt
 RUN python3 -m pip install --no-cache-dir --no-deps -r requirements.txt
 
-ENV NVIDIA_VISIBLE_DEVICES all
-ENV NVIDIA_DRIVER_CAPABILITIES compute
-ENV DEBIAN_FRONTEND dialog
-ENV AA_DOCKER_ENV 1
-
 # Copy application files
 COPY context_chat_backend context_chat_backend
 COPY main.py .
 COPY config.?pu.yaml .
 COPY hwdetect.sh .
 
-ENTRYPOINT ["python3", "main.py"]
+ENTRYPOINT [ "./dockerfile_scripts/entrypoint.sh" ]
diff --git a/config.cpu.yaml b/config.cpu.yaml
index d866725..2f3b7dd 100644
--- a/config.cpu.yaml
+++ b/config.cpu.yaml
@@ -12,6 +12,9 @@ model_download_uri: https://download.nextcloud.com/server/apps/context_chat_back
 
 
 vectordb:
+  pgvector:
+    # 'connection' overrides the env var 'CCB_DB_URL'
+
   chroma:
     is_persistent: true
     # chroma_server_host:
@@ -19,9 +22,6 @@ vectordb:
     # chroma_server_ssl_enabled:
     # chroma_server_api_default_path:
 
-  pgvector:
-    connection: postgresql+psycopg://ccbuser:ccbpass@localhost:5432/ccb
-
   weaviate:
     # auth_client_secret:
     # url: http://localhost:8080
diff --git a/config.gpu.yaml b/config.gpu.yaml
index 1467161..570091e 100644
--- a/config.gpu.yaml
+++ b/config.gpu.yaml
@@ -12,6 +12,9 @@ model_download_uri: https://download.nextcloud.com/server/apps/context_chat_back
 
 
 vectordb:
+  pgvector:
+    # 'connection' overrides the env var 'CCB_DB_URL'
+
   chroma:
     is_persistent: true
     # chroma_server_host:
@@ -19,9 +22,6 @@ vectordb:
     # chroma_server_ssl_enabled:
     # chroma_server_api_default_path:
 
-  pgvector:
-    connection: postgresql+psycopg://ccbuser:ccbpass@localhost:5432/ccb
-
   weaviate:
     # auth_client_secret:
     # url: http://localhost:8080
diff --git a/context_chat_backend/vectordb/pgvector.py b/context_chat_backend/vectordb/pgvector.py
index c75f82c..e1f630b 100644
--- a/context_chat_backend/vectordb/pgvector.py
+++ b/context_chat_backend/vectordb/pgvector.py
@@ -1,3 +1,4 @@
+import os
 from logging import error as log_error
 from typing import Any
 
@@ -23,11 +24,24 @@ class VectorDB(BaseVectorDB):
 	def __init__(self, embedding: Embeddings | None = None, **kwargs):
 		if not embedding:
 			raise DbException('Error: embedding model not provided for pgvector')
-		if 'connection' not in kwargs:
-			raise DbException('Error: connection string not provided for pgvector')
+		if os.getenv('CCB_DB_URL') is None and 'connection' not in kwargs:
+			raise DbException(
+				'Error: Either env var CCB_DB_URL or connection string in the config is required for pgvector'
+			)
 
-		self.client_kwargs = kwargs
 		self.embedding = embedding
+		self.client_kwargs = kwargs
+		# Use connection string from env var if not provided in kwargs
+		self.client_kwargs.update({'connection': str(self.client_kwargs.get('connection', os.environ['CCB_DB_URL']))})
+		# todo
+		self.client_kwargs.update({'connection': 'postgresql+psycopg://ccbuser:ccbpass@localhost:5432/ccb'})
+		print('client kwargs:', self.client_kwargs)
+		print(f'Using connection string: {self.client_kwargs["connection"]}')
+		print('Type of connection string:', type(self.client_kwargs['connection']))
+		engine = sa.create_engine(self.client_kwargs['connection'])
+		with engine.connect() as conn:
+			result = conn.execute(sa.text('select pg_catalog.version()')).fetchall()
+			print(result, flush=True)
 
 	def get_users(self) -> list[str]:
 		engine = sa.create_engine(self.client_kwargs['connection'])
diff --git a/dockerfile_scripts/entrypoint.sh b/dockerfile_scripts/entrypoint.sh
new file mode 100755
index 0000000..3de5536
--- /dev/null
+++ b/dockerfile_scripts/entrypoint.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+source /etc/environment
+"$(dirname $(realpath $0))/pgsql/setup.sh"
+source /etc/environment
+
+python3 -u "$(dirname $(dirname $(realpath $0)))/main.py"
diff --git a/dockerfile_scripts/install_deps.sh b/dockerfile_scripts/install_deps.sh
new file mode 100755
index 0000000..7f46baa
--- /dev/null
+++ b/dockerfile_scripts/install_deps.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+apt-get update
+apt-get install -y --no-install-recommends vim git pciutils
diff --git a/dockerfile_scripts/install_py11.sh b/dockerfile_scripts/install_py11.sh
new file mode 100755
index 0000000..500afe5
--- /dev/null
+++ b/dockerfile_scripts/install_py11.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+apt-get update
+apt-get install -y software-properties-common
+add-apt-repository -y ppa:deadsnakes/ppa
+apt-get update
+apt-get install -y --no-install-recommends python3.11 python3.11-venv python3-pip vim git pciutils
+update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
diff --git a/dockerfile_scripts/pgsql/env b/dockerfile_scripts/pgsql/env
new file mode 100644
index 0000000..e733b6a
--- /dev/null
+++ b/dockerfile_scripts/pgsql/env
@@ -0,0 +1,7 @@
+# Define the PostgreSQL data directory
+BASE_DIR="${APP_PERSISTENT_STORAGE:-/nc_app_context_chat_backend_data}/vector_db_data"
+DATA_DIR="${BASE_DIR}/pgsql"
+
+PG_VERSION=17
+PG_BIN="/usr/lib/postgresql/${PG_VERSION}/bin"
+PG_SQL="/usr/lib/postgresql/${PG_VERSION}/bin/psql"
diff --git a/dockerfile_scripts/pgsql/install.sh b/dockerfile_scripts/pgsql/install.sh
new file mode 100755
index 0000000..3f17dc3
--- /dev/null
+++ b/dockerfile_scripts/pgsql/install.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Stolen from https://github.com/cloud-py-api/flow
+
+set -e
+
+# Environment variables
+source "$(dirname $(realpath $0))/env"
+
+apt-get update
+apt-get install -y curl sudo
+
+# Check if PostgreSQL is installed by checking for the existence of binary files
+if [ -d "$PG_BIN" ]; then
+    echo "PostgreSQL binaries found."
+else
+    echo "PostgreSQL binaries not found."
+    echo "Adding the PostgreSQL APT repository..."
+    VERSION="$(awk -F'=' '/^VERSION_CODENAME=/{ print $NF }' /etc/os-release)"
+    echo "deb http://apt.postgresql.org/pub/repos/apt ${VERSION}-pgdg main" >/etc/apt/sources.list.d/pgdg.list
+    curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor --output /etc/apt/trusted.gpg.d/postgresql.gpg
+    echo "Installing PostgreSQL..."
+    apt-get update && apt-get install -y postgresql-$PG_VERSION postgresql-$PG_VERSION-pgvector
+fi
diff --git a/dockerfile_scripts/pgsql/setup.sh b/dockerfile_scripts/pgsql/setup.sh
new file mode 100755
index 0000000..2903fec
--- /dev/null
+++ b/dockerfile_scripts/pgsql/setup.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Stolen from https://github.com/cloud-py-api/flow
+
+set -e
+
+# Environment variables
+source "$(dirname $(realpath $0))/env"
+
+# Check if CCB_DB_URL is set
+if [ -n "${CCB_DB_URL}" ]; then
+    echo "CCB_DB_URL is set to: $CCB_DB_URL"
+    if [[ "$CCB_DB_URL" != "postgresql+psycopg://"* ]]; then
+        echo "CCB_DB_URL must be a PostgreSQL URL and start with 'postgresql+psycopg://'"
+        exit 1
+    fi
+    exit 0
+fi
+
+# Ensure the directory exists and has the correct permissions
+mkdir -p "$DATA_DIR"
+chown -R postgres:postgres "$DATA_DIR"
+
+if [ ! -d "$DATA_DIR/base" ]; then
+    echo "Initializing the PostgreSQL database..."
+    sudo -u postgres ${PG_BIN}/initdb -D "$DATA_DIR"
+fi
+
+echo "Starting PostgreSQL..."
+sudo -u postgres ${PG_BIN}/pg_ctl -D "$DATA_DIR" -l "${DATA_DIR}/logfile" start
+
+echo "Waiting for PostgreSQL to start..."
+until sudo -u postgres ${PG_SQL} -c "SELECT 1" > /dev/null 2>&1; do
+    sleep 1
+    echo -n "."
+done
+echo "PostgreSQL is up and running."
+
+# Check if the user exists and create if not
+sudo -u postgres $PG_SQL -c "SELECT 1 FROM pg_user WHERE usename = '$CCB_DB_USER'" | grep -q 1 || \
+sudo -u postgres $PG_SQL -c "CREATE USER $CCB_DB_USER WITH PASSWORD '$CCB_DB_PASS';" && \
+sudo -u postgres $PG_SQL -c "ALTER USER $CCB_DB_USER WITH SUPERUSER;"
+
+# Check if the database exists and create if not
+sudo -u postgres $PG_SQL -c "SELECT 1 FROM pg_database WHERE datname = '$CCB_DB_NAME'" | grep -q 1 || \
+sudo -u postgres $PG_SQL -c "CREATE DATABASE $CCB_DB_NAME OWNER $CCB_DB_USER;"
+
+# Check or create the vector extension
+sudo -u postgres $PG_SQL -c "CREATE EXTENSION IF NOT EXISTS vector"
+
+if ! grep -q "^export CCB_DB_URL=" /etc/environment; then
+    echo "export CCB_DB_URL=\"postgresql+psycopg://$CCB_DB_USER:$CCB_DB_PASS@localhost:5432/$CCB_DB_NAME\"" >> /etc/environment
+fi
diff --git a/requirements.in.txt b/requirements.in.txt
index 3dc5115..6aa0450 100644
--- a/requirements.in.txt
+++ b/requirements.in.txt
@@ -7,6 +7,7 @@ httpx
 InstructorEmbedding
 langchain
 langchain-community
+langchain-postgres
 llama_cpp_python
 msg-parser
 odfdo
diff --git a/requirements.txt b/requirements.txt
index 8bac247..7655b40 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -18,7 +18,7 @@ cffi==1.17.1
 chardet==5.2.0
 charset-normalizer==3.4.0
 chroma-hnswlib==0.7.6
-chromadb==0.5.15
+chromadb==0.5.16
 click==8.1.7
 coloredlogs==15.0.1
 cryptography==43.0.3
@@ -32,9 +32,9 @@ durationpy==0.9
 EbookLib==0.17.1
 emoji==2.14.0
 epub2txt==0.1.6
-et-xmlfile==1.1.0
+et_xmlfile==2.0.0
 eval_type_backport==0.2.0
-fastapi==0.115.3
+fastapi==0.115.4
 filelock==3.16.1
 filetype==1.2.0
 flatbuffers==24.3.25
@@ -43,12 +43,12 @@ fsspec==2024.10.0
 google-auth==2.35.0
 googleapis-common-protos==1.56.4
 greenlet==3.1.1
-grpcio==1.67.0
+grpcio==1.67.1
 h11==0.14.0
 httpcore==1.0.6
 httptools==0.6.4
 httpx==0.27.2
-huggingface-hub==0.26.1
+huggingface-hub==0.26.2
 humanfriendly==10.0
 idna==3.10
 importlib_metadata==8.4.0
@@ -62,7 +62,8 @@ jsonpointer==3.0.0
 kubernetes==31.0.0
 langchain==0.3.4
 langchain-community==0.3.3
-langchain-core==0.3.12
+langchain-core==0.3.13
+langchain-postgres==0.0.12
 langchain-text-splitters==0.3.0
 langdetect==1.0.9
 langsmith==0.1.137
@@ -116,10 +117,13 @@ orjson==3.10.10
 overrides==7.7.0
 packaging==24.1
 pandas==2.2.3
+pgvector==0.2.5
 pillow==11.0.0
 posthog==3.7.0
 propcache==0.2.0
 protobuf==3.20.0
+psycopg==3.2.3
+psycopg-pool==3.2.3
 py-cpuinfo==9.0.0
 pyasn1==0.6.1
 pyasn1_modules==0.4.1
@@ -128,18 +132,18 @@ pydantic==2.9.2
 pydantic-settings==2.6.0
 pydantic_core==2.23.4
 Pygments==2.18.0
-pypdf==5.0.1
+pypdf==5.1.0
 PyPika==0.48.9
 pyproject_hooks==1.2.0
 python-dateutil==2.8.2
 python-dotenv==1.0.1
 python-iso639==2024.10.22
 python-magic==0.4.27
-python-multipart==0.0.12
+python-multipart==0.0.16
 python-pptx==1.0.2
 pytz==2024.2
 PyYAML==6.0.2
-RapidFuzz==3.10.0
+RapidFuzz==3.10.1
 regex==2024.9.11
 requests==2.32.3
 requests-oauthlib==2.0.0
@@ -149,12 +153,13 @@ rsa==4.9
 ruamel.yaml==0.18.6
 ruamel.yaml.clib==0.2.12
 safetensors==0.4.5
+setuptools==75.3.0
 shellingham==1.5.4
 six==1.16.0
 sniffio==1.3.1
 soupsieve==2.6
 SQLAlchemy==2.0.36
-starlette==0.41.0
+starlette==0.41.2
 striprtf==0.0.26
 sympy==1.13.1
 tabulate==0.9.0
@@ -162,8 +167,8 @@ tenacity==9.0.0
 tokenizers==0.20.1
 torch==2.5.0
 torchvision==0.20.0
-tqdm==4.66.5
-transformers==4.45.2
+tqdm==4.66.6
+transformers==4.46.0
 triton==3.1.0
 typer==0.12.5
 typing-inspect==0.9.0
@@ -171,7 +176,7 @@ typing_extensions==4.12.2
 tzdata==2024.2
 tzlocal==5.2
 unstructured @ git+https://github.com/kyteinsky/unstructured@d3a404cfb541dae8e16956f096bac99fc05c985b
-unstructured-client==0.26.1
+unstructured-client==0.26.2
 urllib3==2.2.3
 uvicorn==0.32.0
 uvloop==0.21.0
@@ -184,5 +189,5 @@ wrapt==1.16.0
 xlrd==2.0.1
 XlsxWriter==3.2.0
 xmltodict==0.14.2
-yarl==1.16.0
+yarl==1.17.0
 zipp==3.20.2

From f329e0568505da7bff5cd97d53a6a921ca0d0352 Mon Sep 17 00:00:00 2001
From: Anupam Kumar <kyteinsky@gmail.com>
Date: Thu, 31 Oct 2024 02:06:51 +0530
Subject: [PATCH 2/4] fix: enforce UTF8 encoding in postgres setup

Signed-off-by: Anupam Kumar <kyteinsky@gmail.com>
---
 context_chat_backend/vectordb/pgvector.py |  9 ---------
 dockerfile_scripts/pgsql/setup.sh         | 19 +++++++++++++++----
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/context_chat_backend/vectordb/pgvector.py b/context_chat_backend/vectordb/pgvector.py
index e1f630b..b4ee9d5 100644
--- a/context_chat_backend/vectordb/pgvector.py
+++ b/context_chat_backend/vectordb/pgvector.py
@@ -33,15 +33,6 @@ def __init__(self, embedding: Embeddings | None = None, **kwargs):
 		self.client_kwargs = kwargs
 		# Use connection string from env var if not provided in kwargs
 		self.client_kwargs.update({'connection': str(self.client_kwargs.get('connection', os.environ['CCB_DB_URL']))})
-		# todo
-		self.client_kwargs.update({'connection': 'postgresql+psycopg://ccbuser:ccbpass@localhost:5432/ccb'})
-		print('client kwargs:', self.client_kwargs)
-		print(f'Using connection string: {self.client_kwargs["connection"]}')
-		print('Type of connection string:', type(self.client_kwargs['connection']))
-		engine = sa.create_engine(self.client_kwargs['connection'])
-		with engine.connect() as conn:
-			result = conn.execute(sa.text('select pg_catalog.version()')).fetchall()
-			print(result, flush=True)
 
 	def get_users(self) -> list[str]:
 		engine = sa.create_engine(self.client_kwargs['connection'])
diff --git a/dockerfile_scripts/pgsql/setup.sh b/dockerfile_scripts/pgsql/setup.sh
index 2903fec..f379d59 100755
--- a/dockerfile_scripts/pgsql/setup.sh
+++ b/dockerfile_scripts/pgsql/setup.sh
@@ -7,13 +7,19 @@ set -e
 # Environment variables
 source "$(dirname $(realpath $0))/env"
 
-# Check if CCB_DB_URL is set
-if [ -n "${CCB_DB_URL}" ]; then
-    echo "CCB_DB_URL is set to: $CCB_DB_URL"
+# Check if EXTERNAL_DB is set
+if [ -n "${EXTERNAL_DB}" ]; then
+    CCB_DB_URL="${EXTERNAL_DB}"
+    echo "Using EXTERNAL_DB, CCB_DB_URL is set to: $CCB_DB_URL"
+
     if [[ "$CCB_DB_URL" != "postgresql+psycopg://"* ]]; then
         echo "CCB_DB_URL must be a PostgreSQL URL and start with 'postgresql+psycopg://'"
         exit 1
     fi
+
+    if ! grep -q "^export EXTERNAL_DB=" /etc/environment; then
+        echo "export EXTERNAL_DB=\"$EXTERNAL_DB\"" >> /etc/environment
+    fi
     exit 0
 fi
 
@@ -23,7 +29,7 @@ chown -R postgres:postgres "$DATA_DIR"
 
 if [ ! -d "$DATA_DIR/base" ]; then
     echo "Initializing the PostgreSQL database..."
-    sudo -u postgres ${PG_BIN}/initdb -D "$DATA_DIR"
+    sudo -u postgres ${PG_BIN}/initdb -D "$DATA_DIR" -E UTF8
 fi
 
 echo "Starting PostgreSQL..."
@@ -36,6 +42,11 @@ until sudo -u postgres ${PG_SQL} -c "SELECT 1" > /dev/null 2>&1; do
 done
 echo "PostgreSQL is up and running."
 
+if [ -n "${CCB_DB_URL}" ]; then
+    echo "CCB_DB_URL is already set. Skipping database setup."
+    exit 0
+fi
+
 # Check if the user exists and create if not
 sudo -u postgres $PG_SQL -c "SELECT 1 FROM pg_user WHERE usename = '$CCB_DB_USER'" | grep -q 1 || \
 sudo -u postgres $PG_SQL -c "CREATE USER $CCB_DB_USER WITH PASSWORD '$CCB_DB_PASS';" && \

From c5f4247dbb2c96fe4968ac2858436ea6615f8390 Mon Sep 17 00:00:00 2001
From: Anupam Kumar <kyteinsky@gmail.com>
Date: Thu, 31 Oct 2024 02:14:52 +0530
Subject: [PATCH 3/4] update readme

Signed-off-by: Anupam Kumar <kyteinsky@gmail.com>
---
 README.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index e74c5d2..46c26b9 100644
--- a/README.md
+++ b/README.md
@@ -38,8 +38,11 @@ Install the given apps for Context Chat to work as desired **in the given order*
 5. Copy example.env to .env and fill in the variables
 6. Ensure the config file at `persistent_storage/config.yaml` points to the correct config file (cpu vs gpu). If you're unsure, delete it. It will be recreated upon launching the application. The default is to point to the gpu config.
 7. Configure `persistent_storage/config.yaml` for the model name, model type and its parameters (which also includes model file's path and model id as per requirements, see example config)
-8. `./main.py`
-9. [Follow the below steps to register the app in the app ecosystem](#register-as-an-ex-app)
+8. Setup postgresql externally or use `dockerfile_scripts/pgsql/install.sh` to install it on a Debian-family system.
+9. Set the env var `EXTERNAL_DB` or the `connection` key in the `pgvector` config to the postgresql connection string if you're using an external database.
+10. Start the database (see `dockerfile_scripts/pgsql/setup.sh` for an example)
+11. `./main.py`
+12. [Follow the below steps to register the app in the app ecosystem](#register-as-an-ex-app)
 
 ## Complex Install (with docker)
 

From 024589d80160da2c1abc9b861eb5124b03e1ba75 Mon Sep 17 00:00:00 2001
From: Anupam Kumar <kyteinsky@gmail.com>
Date: Thu, 31 Oct 2024 02:24:58 +0530
Subject: [PATCH 4/4] update integration-test

Signed-off-by: Anupam Kumar <kyteinsky@gmail.com>
---
 .github/workflows/integration-test.yml | 6 ++++--
 dockerfile_scripts/pgsql/env           | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index d969d97..494216e 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -41,7 +41,7 @@ jobs:
       fail-fast: false
       matrix:
         php-versions: [ '8.1' ]
-        databases: [ 'sqlite' ]
+        databases: [ 'pgsql' ]
         server-versions: [ 'master' ]
 
     name: Integration test on ${{ matrix.server-versions }} php@${{ matrix.php-versions }}
@@ -49,6 +49,8 @@ jobs:
     env:
       MYSQL_PORT: 4444
       PGSQL_PORT: 4445
+      # use the same db for ccb and nextcloud
+      CCB_DB_URL: postgresql+psycopg://root:rootpassword@localhost:4445/nextcloud
 
     services:
       mysql:
@@ -59,7 +61,7 @@ jobs:
           MYSQL_ROOT_PASSWORD: rootpassword
         options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5
       postgres:
-        image: postgres
+        image: pgvector/pgvector:pg17
         ports:
           - 4445:5432/tcp
         env:
diff --git a/dockerfile_scripts/pgsql/env b/dockerfile_scripts/pgsql/env
index e733b6a..699cb40 100644
--- a/dockerfile_scripts/pgsql/env
+++ b/dockerfile_scripts/pgsql/env
@@ -1,5 +1,5 @@
 # Define the PostgreSQL data directory
-BASE_DIR="${APP_PERSISTENT_STORAGE:-/nc_app_context_chat_backend_data}/vector_db_data"
+BASE_DIR="${APP_PERSISTENT_STORAGE:-persistent_storage}/vector_db_data"
 DATA_DIR="${BASE_DIR}/pgsql"
 
 PG_VERSION=17