Add SQLite config for Docker

container/postgres/.gitignore

@@ -0,0 +1,2 @@
+alembic/*
+alembic.ini

container/postgres/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.11
+
+RUN pip install --no-cache-dir --upgrade "git+https://github.com/simonsobs/librarian.git@librarian-v2"
+RUN pip install --no-cache-dir --upgrade "psycopg[binary,pool]"
+
+COPY ./server_config.json server_config.json
+COPY ./background_config.json background_config.json
+COPY ./alembic.ini alembic.ini
+COPY ./alembic alembic
+COPY ./setup_vars.sh setup_vars.sh
+
+RUN mkdir -p /tmp/store/libstore/store
+RUN mkdir -p /tmp/store/libstore/staging
+RUN mkdir -p /tmp/store/libclone/store
+RUN mkdir -p /tmp/store/libclone/staging
+RUN mkdir -p /volumes/database
+
+CMD ["librarian-server-start"]
+

container/postgres/README.md

@@ -0,0 +1,31 @@
+This is a more complex librarian setup, and so requires a little more thought when setting
+it up.
+
+A few things to keep in mind when deploying the librarian:
+
+- This is inherently not secure by default; you will need to change the administrator
+  password for the librarian and potentially provision further accounts.
+
+- You will need to mark both the hostname of the docker container and of your
+  local machine as available for local transfers.
+
+- You will need to make sure that the local store locations are mounted
+  transparently by the docker container (i.e. you should have /path/to/store
+  the same inside and outside of the container).
+
+Specifically for this librarian, as we have the database running in a different Docker
+container, we'll need to make sure that this is up and running before running anything
+like a database migration.
+
+So, you will need to:
+
+- `bash pre_docker_setup.sh`
+- `docker compose build`
+- `docker compose up` (this will cause errors!)
+- `docker ps` -> grab the ID of the running librarian server instance
+- `docker exec -ti {ID} bash` -> into the container
+- `source setup_vars.sh`
+- `librarian-server-setup`
+- `exit`
+- Restart the docker container.
+- Your application should now be happily running.

container/postgres/background_config.json

@@ -0,0 +1,13 @@
+{
+    "create_local_clone": [
+        {
+            "task_name": "Local cloner",
+            "every": "00:05:00",
+            "age_in_days": 7,
+            "clone_from": "store",
+            "clone_to": "clone"
+        }
+    ]
+}
+
+

container/postgres/docker-compose.yml

@@ -0,0 +1,61 @@
+version: "3.11"
+
+services:
+  librarian-database:
+    image: postgres:16
+    restart: always
+    container_name: "librarian-database"
+    user: postgres
+    volumes:
+      - "database:/var/lib/postgresql/data"
+    environment:
+      POSTGRES_USER: "postgres"
+      POSTGRES_PASSWORD: "secret"
+      POSTGRES_DB: "librarian"
+    expose:
+      - 5432
+    networks:
+      - "librarian-network"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready"]
+      interval: 3s
+      timeout: 5s
+      retries: 5
+
+  librarian-server:
+    hostname: "example-librarian-hostname"
+    build:
+      context: "."
+      dockerfile: "Dockerfile"
+    container_name: "librarian-postgres"
+    ports:
+      - 127.0.0.1:21108:21108
+    stdin_open: true
+    tty: true
+    volumes:
+      - "database:/volumes/database"
+      - type: "bind"
+        source: "/tmp/store"
+        target: "/tmp/store"
+    environment:
+      - LIBRARIAN_CONFIG_PATH=server_config.json
+      - LIBRARIAN_SERVER_DATABASE_USER=postgres
+      - LIBRARIAN_SERVER_DATABASE_PASSWORD=secret
+      - LIBRARIAN_BACKGROUND_CONFIG=background_config.json
+      - LIBRARIAN_SERVER_DATABASE_DRIVER=sqlite
+      - LIBRARIAN_SERVER_DATABASE=/volumes/database/database.db
+      - LIBRARIAN_SERVER_PORT=21108
+    depends_on:
+      librarian-database:
+        condition: service_healthy
+    networks:
+      - "librarian-network"
+
+volumes:
+  database:
+
+networks:
+  librarian-network:
+
+
+

container/postgres/pre_docker_setup.sh

@@ -0,0 +1,14 @@
+# Set up paths
+
+mkdir -p /tmp/store/libstore/staging
+mkdir -p /tmp/store/libstore/store
+mkdir -p /tmp/store/libclone/staging
+mkdir -p /tmp/store/libclone/store
+
+# We need to copy in the alembic to here, so we can run the
+# database migration.
+
+cp -r ../../alembic .
+cp ../../alembic.ini .
+
+echo "You need to change the value of YOUR_HOSTNAME to: $(hostname) in server_config.json"

container/postgres/server_config.json

@@ -0,0 +1,47 @@
+{   
+    "database_driver": "postgresql+psycopg",
+    "database_port": "5432",
+    "database": "librarian",
+    "database_host": "librarian-database",
+    "log_level": "DEBUG",
+    "displayed_site_name": "Docker Example",
+    "displayed_site_description": "An example docker deployment of the librarian.",
+    "add_stores": [
+        {
+            "store_name": "store",
+            "store_type": "local",
+            "ingestable": true,
+            "store_data": {
+                "staging_path": "/tmp/store/libstore/staging",
+                "store_path": "/tmp/store/libstore/store"
+            },
+            "transfer_manager_data": {
+                "local": {
+                    "available": true,
+                    "hostnames": [
+                        "borrowj-adapter.physics.upenn.edu",
+                        "example-librarian-hostname"
+                    ]
+                }
+            }
+        },
+        {
+            "store_name": "clone",
+            "store_type": "local",
+            "ingestable": false,
+            "store_data": {
+                "staging_path": "/tmp/store/libclone/staging",
+                "store_path": "/tmp/store/libclone/store"
+            },
+            "transfer_manager_data": {
+                "local": {
+                    "available": true,
+                    "hostnames": [
+                        "borrowj-adapter.physics.upenn.edu",
+                        "example-librarian-hostname"
+                    ]
+                }
+            }
+        }
+    ]
+}

container/postgres/setup_vars.sh

@@ -0,0 +1,2 @@
+export LIBRARIAN_SERVER_DATABASE_USER="postgres"
+export LIBRARIAN_SERVER_DATABASE_PASSWORD="secret"

container/sqlite/Dockerfile

@@ -14,8 +14,10 @@ RUN cat server_config.json
 RUN echo "Background config:"
 RUN cat background_config.json
 
-RUN mkdir -p /mounts/stores/libstore
-RUN mkdir -p /mounts/stores/libclone
+RUN mkdir -p /tmp/store/libstore/store
+RUN mkdir -p /tmp/store/libstore/staging
+RUN mkdir -p /tmp/store/libclone/store
+RUN mkdir -p /tmp/store/libclone/staging
 RUN mkdir -p /volumes/database
 
 ARG LIBRARIAN_CONFIG_PATH

container/sqlite/README.md

@@ -0,0 +1,21 @@
+This is a very simple setup for the librarian, using a local SQLite database
+as the core librarian database. Production setups may want to use the
+postgres container.
+
+You will need to run the `pre_docker_setup.sh` script to put things in the right
+place before, edit server_config.json a little, and run:
+
+- `docker compose build`
+- `docker compose up`
+
+A few things to keep in mind when deploying the librarian:
+
+- This is inherently not secure by default; you will need to change the administrator
+  password for the librarian and potentially provision further accounts.
+
+- You will need to mark both the hostname of the docker container and of your
+  local machine as available for local transfers.
+
+- You will need to make sure that the local store locations are mounted
+  transparently by the docker container (i.e. you should have /path/to/store
+  the same inside and outside of the container).

container/sqlite/background_config.json

@@ -2,7 +2,7 @@
     "create_local_clone": [
         {
             "task_name": "Local cloner",
-            "every": "06:00:00",
+            "every": "00:05:00",
             "age_in_days": 7,
             "clone_from": "store",
             "clone_to": "clone"

container/sqlite/docker-compose.yml

@@ -16,7 +16,9 @@ services:
       - 127.0.0.1:21108:21108
     volumes:
       - "database:/volumes/database"
-      - "store:/mounts/stores"
+      - type: "bind"
+        source: "/tmp/store"
+        target: "/tmp/store"
     environment:
       - LIBRARIAN_CONFIG_PATH=server_config.json
       - LIBRARIAN_BACKGROUND_CONFIG=background_config.json
@@ -25,6 +27,5 @@ services:
       - LIBRARIAN_SERVER_PORT=21108
 
 volumes:
-  store: 
   database:
 

container/sqlite/pre_docker_setup.sh

@@ -1,5 +1,14 @@
+# Set up paths
+
+mkdir -p /tmp/store/libstore/staging
+mkdir -p /tmp/store/libstore/store
+mkdir -p /tmp/store/libclone/staging
+mkdir -p /tmp/store/libclone/store
+
 # We need to copy in the alembic to here, so we can run the
 # database migration.
 
 cp -r ../../alembic .
 cp ../../alembic.ini .
+
+echo "You need to change the value of YOUR_HOSTNAME to: $(hostname) in server_config.json"