diff --git a/container/postgres/.gitignore b/container/postgres/.gitignore new file mode 100644 index 0000000..fe64225 --- /dev/null +++ b/container/postgres/.gitignore @@ -0,0 +1,2 @@ +alembic/* +alembic.ini diff --git a/container/postgres/Dockerfile b/container/postgres/Dockerfile new file mode 100644 index 0000000..6c9bb83 --- /dev/null +++ b/container/postgres/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.11 + +RUN pip install --no-cache-dir --upgrade "git+https://github.com/simonsobs/librarian.git@librarian-v2" +RUN pip install --no-cache-dir --upgrade "psycopg[binary,pool]" + +COPY ./server_config.json server_config.json +COPY ./background_config.json background_config.json +COPY ./alembic.ini alembic.ini +COPY ./alembic alembic +COPY ./setup_vars.sh setup_vars.sh + +RUN mkdir -p /tmp/store/libstore/store +RUN mkdir -p /tmp/store/libstore/staging +RUN mkdir -p /tmp/store/libclone/store +RUN mkdir -p /tmp/store/libclone/staging +RUN mkdir -p /volumes/database + +CMD ["librarian-server-start"] + diff --git a/container/postgres/background_config.json b/container/postgres/background_config.json new file mode 100644 index 0000000..f7985cf --- /dev/null +++ b/container/postgres/background_config.json @@ -0,0 +1,13 @@ +{ + "create_local_clone": [ + { + "task_name": "Local cloner", + "every": "00:05:00", + "age_in_days": 7, + "clone_from": "store", + "clone_to": "clone" + } + ] +} + + diff --git a/container/postgres/docker-compose.yml b/container/postgres/docker-compose.yml new file mode 100644 index 0000000..fd0ad1e --- /dev/null +++ b/container/postgres/docker-compose.yml @@ -0,0 +1,61 @@ +version: "3.11" + +services: + librarian-database: + image: postgres:16 + restart: always + container_name: "librarian-database" + user: postgres + volumes: + - "database:/var/lib/postgresql/data" + environment: + POSTGRES_USER: "postgres" + POSTGRES_PASSWORD: "secret" + POSTGRES_DB: "librarian" + expose: + - 5432 + networks: + - "librarian-network" + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 3s + timeout: 5s + retries: 5 + + librarian-server: + hostname: "example-librarian-hostname" + build: + context: "." + dockerfile: "Dockerfile" + container_name: "librarian-postgres" + ports: + - 127.0.0.1:21108:21108 + stdin_open: true + tty: true + volumes: + - "database:/volumes/database" + - type: "bind" + source: "/tmp/store" + target: "/tmp/store" + environment: + - LIBRARIAN_CONFIG_PATH=server_config.json + - LIBRARIAN_SERVER_DATABASE_USER=postgres + - LIBRARIAN_SERVER_DATABASE_PASSWORD=secret + - LIBRARIAN_BACKGROUND_CONFIG=background_config.json + - LIBRARIAN_SERVER_DATABASE_DRIVER=sqlite + - LIBRARIAN_SERVER_DATABASE=/volumes/database/database.db + - LIBRARIAN_SERVER_PORT=21108 + depends_on: + librarian-database: + condition: service_healthy + networks: + - "librarian-network" + +volumes: + database: + +networks: + librarian-network: + + + diff --git a/container/postgres/pre_docker_setup.sh b/container/postgres/pre_docker_setup.sh new file mode 100644 index 0000000..a3dddf0 --- /dev/null +++ b/container/postgres/pre_docker_setup.sh @@ -0,0 +1,5 @@ +# We need to copy in the alembic to here, so we can run the +# database migration. + +cp -r ../../alembic . +cp ../../alembic.ini . diff --git a/container/postgres/server_config.json b/container/postgres/server_config.json new file mode 100644 index 0000000..11e3a75 --- /dev/null +++ b/container/postgres/server_config.json @@ -0,0 +1,47 @@ +{ + "database_driver": "postgresql+psycopg", + "database_port": "5432", + "database": "librarian", + "database_host": "librarian-database", + "log_level": "DEBUG", + "displayed_site_name": "Docker Example", + "displayed_site_description": "An example docker deployment of the librarian.", + "add_stores": [ + { + "store_name": "store", + "store_type": "local", + "ingestable": true, + "store_data": { + "staging_path": "/tmp/store/libstore/staging", + "store_path": "/tmp/store/libstore/store" + }, + "transfer_manager_data": { + "local": { + "available": true, + "hostnames": [ + "borrowj-adapter.physics.upenn.edu", + "example-librarian-hostname" + ] + } + } + }, + { + "store_name": "clone", + "store_type": "local", + "ingestable": false, + "store_data": { + "staging_path": "/tmp/store/libclone/staging", + "store_path": "/tmp/store/libclone/store" + }, + "transfer_manager_data": { + "local": { + "available": true, + "hostnames": [ + "borrowj-adapter.physics.upenn.edu", + "example-librarian-hostname" + ] + } + } + } + ] +} diff --git a/container/postgres/setup_vars.sh b/container/postgres/setup_vars.sh new file mode 100644 index 0000000..4d42a88 --- /dev/null +++ b/container/postgres/setup_vars.sh @@ -0,0 +1,2 @@ +export LIBRARIAN_SERVER_DATABASE_USER="postgres" +export LIBRARIAN_SERVER_DATABASE_PASSWORD="secret" diff --git a/container/sqlite/README.md b/container/sqlite/README.md new file mode 100644 index 0000000..8cbd17e --- /dev/null +++ b/container/sqlite/README.md @@ -0,0 +1,15 @@ +This is a very simple setup for the librarian, using a local SQLite database +as the core librarian database. Production setups may want to use the +postgres container. + +A few things to keep in mind when deploying the librarian: + +- This is inherently not secure by default; you will need to change the administrator + password for the librarian and potentially provision further accounts. + +- You will need to mark both the hostname of the docker container and of your + local machine as available for local transfers. + +- You will need to make sure that the local store locations are mounted + transparently by the docker container (i.e. you should have /path/to/store + the same inside and outside of the container). \ No newline at end of file diff --git a/container/wait-for-it.sh b/container/wait-for-it.sh deleted file mode 100755 index f0a04f7..0000000 --- a/container/wait-for-it.sh +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env bash -# Script copied from https://github.com/vishnubob/wait-for-it/blob/master/wait-for-it.sh. -# Use this script to test if a given TCP host/port are available - -WAITFORIT_cmdname=${0##*/} - -echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} - -wait_for() -{ - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - else - echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" - fi - WAITFORIT_start_ts=$(date +%s) - while : - do - if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then - nc -z $WAITFORIT_HOST $WAITFORIT_PORT - WAITFORIT_result=$? - else - (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 - WAITFORIT_result=$? - fi - if [[ $WAITFORIT_result -eq 0 ]]; then - WAITFORIT_end_ts=$(date +%s) - echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" - break - fi - sleep 1 - done - return $WAITFORIT_result -} - -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $WAITFORIT_QUIET -eq 1 ]]; then - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - else - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - fi - WAITFORIT_PID=$! - trap "kill -INT -$WAITFORIT_PID" INT - wait $WAITFORIT_PID - WAITFORIT_RESULT=$? - if [[ $WAITFORIT_RESULT -ne 0 ]]; then - echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - fi - return $WAITFORIT_RESULT -} - -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - WAITFORIT_hostport=(${1//:/ }) - WAITFORIT_HOST=${WAITFORIT_hostport[0]} - WAITFORIT_PORT=${WAITFORIT_hostport[1]} - shift 1 - ;; - --child) - WAITFORIT_CHILD=1 - shift 1 - ;; - -q | --quiet) - WAITFORIT_QUIET=1 - shift 1 - ;; - -s | --strict) - WAITFORIT_STRICT=1 - shift 1 - ;; - -h) - WAITFORIT_HOST="$2" - if [[ $WAITFORIT_HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - WAITFORIT_HOST="${1#*=}" - shift 1 - ;; - -p) - WAITFORIT_PORT="$2" - if [[ $WAITFORIT_PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - WAITFORIT_PORT="${1#*=}" - shift 1 - ;; - -t) - WAITFORIT_TIMEOUT="$2" - if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - WAITFORIT_TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - WAITFORIT_CLI=("$@") - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done - -if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi - -WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} -WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} -WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} -WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} - -# Check to see if timeout is from busybox? -WAITFORIT_TIMEOUT_PATH=$(type -p timeout) -WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) - -WAITFORIT_BUSYTIMEFLAG="" -if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then - WAITFORIT_ISBUSY=1 - # Check if busybox timeout uses -t flag - # (recent Alpine versions don't support -t anymore) - if timeout &>/dev/stdout | grep -q -e '-t '; then - WAITFORIT_BUSYTIMEFLAG="-t" - fi -else - WAITFORIT_ISBUSY=0 -fi - -if [[ $WAITFORIT_CHILD -gt 0 ]]; then - wait_for - WAITFORIT_RESULT=$? - exit $WAITFORIT_RESULT -else - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - wait_for_wrapper - WAITFORIT_RESULT=$? - else - wait_for - WAITFORIT_RESULT=$? - fi -fi - -if [[ $WAITFORIT_CLI != "" ]]; then - if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then - echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" - exit $WAITFORIT_RESULT - fi - exec "${WAITFORIT_CLI[@]}" -else - exit $WAITFORIT_RESULT -fi diff --git a/librarian_server_scripts/librarian_server_setup.py b/librarian_server_scripts/librarian_server_setup.py index a701d6d..03a277d 100644 --- a/librarian_server_scripts/librarian_server_setup.py +++ b/librarian_server_scripts/librarian_server_setup.py @@ -196,7 +196,7 @@ def main(): try: conn.execute( text( - f"CREATE USER {args.librarian_db_user} WITH PASSWORD '{args.librarian_db_password}'" + f"CREATE USER {args.librarian_db_user} WITH PASSWORD {args.librarian_db_password}" ) ) conn.execute(text(f"GRANT libserver TO {args.librarian_db_user}"))