Merge pull request #447 from hotosm/feature/ci

Merge: Optimise dockerfiles, fix ci, release images

.dockerignore

@@ -0,0 +1,22 @@
+# Ignore everything
+**
+
+# Allow files and directories
+
+# Underpass
+!src
+!config
+!setup
+!docker/bzip2.pc
+!autogen.sh
+!configure.ac
+!Makefile.am
+!m4
+!dist
+!docs
+!ABOUT-NLS
+!config.rpath
+!docker/ci-entrypoint.sh
+
+# API
+!python

.github/workflows/release_img.yml

@@ -0,0 +1,29 @@
+name: 🔧 Build Release Images
+
+on:
+  release:
+    types: [published]
+  # Allow manual trigger
+  workflow_dispatch:
+
+jobs:
+  underpass-build:
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2
+    with:
+      dockerfile: docker/underpass.dockerfile
+      build_target: prod
+      image_name: ghcr.io/${{ github.repository }}
+
+  api-build:
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2
+    with:
+      dockerfile: docker/underpass-api.dockerfile
+      build_target: prod
+      image_name: ghcr.io/${{ github.repository }}/api
+
+  ui-build:
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2
+    with:
+      dockerfile: docker/underpass-ui.dockerfile
+      build_target: prod
+      image_name: ghcr.io/${{ github.repository }}/ui

.github/workflows/tests.yml

@@ -1,29 +1,25 @@
-name: 🧪 Build and test
+name: 🧪 Build and Test
 
 on:
   push:
     branches:
       - master
+  pull_request:
+    branches:
+      - master
     paths:
       - src/**
       - .github/workflows/**
       - docker/**
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      # Relies on the master branch built docker image for build-deps
-      - name: Start Docker Compose & build
-        run: cd docker && docker-compose up -d underpass
-      # Build and run tests
-      - name: Build and run tests 
-        run: |
-          docker-compose -f docker-compose.yml exec -T underpass sh -c "cd /code/build && make check -j `nproc`"
-          exit_code=$?
-          if [ $exit_code -ne 0 ]; then
-            echo "Test failed with exit code $exit_code"
-            exit $exit_code
-          fi
+  build-and-test:
+    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.2.1
+    with:
+      image_name: ghcr.io/${{ github.repository }}
+      build_dockerfile: docker/underpass.dockerfile
+      compose_service: underpass
+      compose_command: echo "Tests complete."
+      # TODO update postgis image to use github repo var ${{ vars.POSTGIS_TAG }}
+      cache_extra_imgs: |
+        "docker.io/postgis/postgis:15-3.3-alpine"

.github/workflows/tests/pr_payload.json

@@ -0,0 +1,10 @@
+{
+  "pull_request": {
+    "head": {
+      "ref": "feat/some-new-thing"
+    },
+    "base": {
+      "ref": "master"
+    }
+  }
+}

.github/workflows/tests/push_payload.json

@@ -0,0 +1,4 @@
+{
+  "base_ref ": "master",
+  "ref": "refs/heads/master"
+}

.github/workflows/tests/test_ci.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -e
+
+########################################
+# Note: run this from the repo root.
+########################################
+
+# Tests
+act pull_request -W .github/workflows/tests.yml \
+    -e .github/workflows/tests/pr_payload.json
+
+# Docs
+act push -W .github/workflows/docs.yml \
+    -e .github/workflows/tests/pr_payload.json
+
+# Release
+act release -W .github/workflows/release_img.yml \
+    -e .github/workflows/tests/push_payload.json

ci/ci_local.sh

@@ -19,24 +19,12 @@ cd ${TEMP_DIR}
 make distclean -j `nproc` || true
 make clean -j `nproc` || true
 
-DOCKER_DIR="${TEMP_DIR}"
-DOCKER_COMPOSE_FILE="${DOCKER_DIR}/docker-compose.yml"
-DOCKER_BASE_COMMAND="docker-compose -f ${DOCKER_COMPOSE_FILE}"
+# Run CI
+APP_VERSION=ci docker compose run underpass --exit-code-from=underpass
 
-# Run the composition
-${DOCKER_BASE_COMMAND} up -d
+# Shut down containers
+APP_VERSION=ci docker compose down
 
-# Build Underpass Library and Binaries
-${DOCKER_BASE_COMMAND} exec -T underpass sh -c "cd /code && git clean -fx && git clean -f -d && ./autogen.sh && (rm -rf build || true) && mkdir build && cd build && ../configure --enable-shared && make -j `nproc`"
-
-# Build and Run Underpass Tests - broken: alway succeeds
-${DOCKER_BASE_COMMAND} exec -T underpass sh -c "cd /code/build/src/testsuite/libunderpass.all && make check -j `nproc`"
-
-# Comment the cleanup lines below or exit here if you want to run additional
-# tests from a console in the temp container, for instance with:
-${DOCKER_BASE_COMMAND} exec underpass bash
-
-${DOCKER_BASE_COMMAND} down
 echo "Remove temporary folder ${TEMP_DIR}"
 sudo rm -rf ${TEMP_DIR}
 

docker-compose.yml

@@ -22,11 +22,10 @@ version: "3"
 services:
   # Database
   postgis:
-    image: postgis/postgis:15-3.3
-    # image: arm64v8/postgres:15.3-alpine3.18
+    image: postgis/postgis:${POSTGIS_TAG:-15-3.3-alpine}
     container_name: "underpass_postgis"
     ports:
-      - "5439:5432"
+      - "${DB_PORT:-5439}:5432"
     environment:
       - POSTGRES_DB=underpass
       - POSTGRES_USER=underpass
@@ -44,44 +43,61 @@ services:
 
   # Underpass
   underpass:
-    container_name: "underpass"
+    image: "ghcr.io/hotosm/underpass:${APP_VERSION:-debug}"
+    container_name: underpass
     build:
      context: .
      dockerfile: docker/underpass.dockerfile
+     target: debug
+     args:
+        APP_VERSION: ${APP_VERSION:-debug}
+    depends_on: [postgis]
     environment:
       - REPLICATOR_UNDERPASS_DB_URL=underpass:underpass@postgis/underpass
     command: tail -f /dev/null
     volumes:
       - ./replication:/code/build/replication
+  # # Mount code
   #   - ./:/code
     networks:
       internal:
 
   # Underpass API
-  underpass-api:
-    container_name: "underpass_api"
+  api:
+    image: "ghcr.io/hotosm/underpass/api:${APP_VERSION:-debug}"
+    container_name: underpass_api
     build:
      context: .
      dockerfile: docker/underpass-api.dockerfile
-    # volumes:
-    #   - ./python:/code/api
+     target: debug
+     args:
+        APP_VERSION: ${APP_VERSION:-debug}
+    depends_on: [underpass]
+    volumes:
+      - ./python:/code
     ports:
-      - "8000:8000"
+      - "${API_PORT:-8000}:8000"
     networks:
       internal:
     environment:
       - UNDERPASS_API_DB=postgresql://underpass:underpass@postgis/underpass
 
   # Underpass UI
-  underpass-ui:
-    container_name: "underpass_ui"
+  ui:
+    image: "ghcr.io/hotosm/underpass/ui:${APP_VERSION:-debug}"
+    container_name: underpass_ui
     build:
      context: .
      dockerfile: docker/underpass-ui.dockerfile
+     target: debug
+     args:
+        APP_VERSION: ${APP_VERSION:-debug}
+    # # Mount underpass-ui repo
     # volumes:
-    #   - js:/code
+    #   - ../underpass-ui/src:/code/src
+    #   - ../underpass-ui/playground:/code/playground
     ports:
-      - "5000:5000"
+      - "${UI_PORT:-5000}:5000"
     networks:
       internal:
 

docker/ci-entrypoint.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -euo pipefail
+
+yellow_echo() {
+    local message="$1"
+    local separator="--------------------------------------------------------"
+    local sep_length=${#separator}
+    local pad_length=$(( (sep_length - ${#message}) / 2 ))
+    local pad=""
+
+    for ((i=0; i<pad_length; i++)); do
+        pad="$pad "
+    done
+
+    echo ""
+    echo -e "\e[0;33m$separator\e[0m"
+    echo -e "\e[0;33m$pad$message$pad\e[0m"
+    echo -e "\e[0;33m$separator\e[0m"
+    echo ""
+}
+
+# Generate makefiles
+yellow_echo "Generating Makefiles"
+./autogen.sh
+
+echo
+echo "Entering 'build' directory"
+cd build
+
+# Test build works
+yellow_echo "Building Underpass"
+../configure
+make -j $(nproc)
+make install
+
+# Run tests
+yellow_echo "Running Tests"
+cd /code/build/src/testsuite/libunderpass.all
+make check -j `nproc`

docker/underpass-api.dockerfile

@@ -1,20 +1,85 @@
 FROM python:3.9
 
-LABEL maintainer="Humanitarian OpenStreetMap Team" Description="This image provides the Underpass API" Vendor="HOT" Version="dev"
+FROM docker.io/python:${PYTHON_TAG}-slim-bookworm as base
+ARG APP_VERSION
+ARG COMMIT_REF
+LABEL org.hotosm.underpass.app-name="underpass-api" \
+      org.hotosm.underpass.app-version="${APP_VERSION}" \
+      org.hotosm.underpass.git-commit-ref="${COMMIT_REF:-none}" \
+      org.hotosm.underpass.maintainer="sysadmin@hotosm.org"
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends "locales" "ca-certificates" \
+    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
+    && rm -rf /var/lib/apt/lists/* \
+    && update-ca-certificates
+# Set locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
 WORKDIR /code
 
 RUN apt-get update && apt-get -y install \ 
     postgresql \
     libpq-dev
 
-COPY ./python/dbapi /code/api/dbapi
-COPY ./python/restapi /code/api/restapi
+FROM base as build
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends \
+        "build-essential" \
+        "libpq-dev" \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /opt/python
+COPY python/dbapi/requirements.txt /opt/python/requirements.txt
+COPY python/restapi/requirements.txt /opt/python/requirements2.txt
+RUN pip install --user --no-warn-script-location --no-cache-dir \
+    -r /opt/python/requirements.txt -r /opt/python/requirements2.txt
 
-RUN pip3 install -r /code/api/dbapi/requirements.txt
-RUN pip3 install -r /code/api/restapi/requirements.txt
+FROM base as runtime
+ARG PYTHON_TAG
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONFAULTHANDLER=1 \
+    PATH="/home/appuser/.local/bin:$PATH" \
+    PYTHONPATH="/opt/restapi" \
+    PYTHON_LIB="/home/appuser/.local/lib/python$PYTHON_TAG/site-packages" \
+    SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \
+    REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \
+    CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends \
+        "postgresql-client" \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=build \
+    /root/.local \
+    /home/appuser/.local
+COPY /python/dbapi /code/dbapi
+COPY /python/restapi /code/restapi
+WORKDIR /code/restapi
+# Add non-root user, permissions
+RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser \
+    && chown -R appuser:appuser /code /home/appuser
+# Change to non-root user
+USER appuser
+# Add Healthcheck
+HEALTHCHECK --start-period=10s --interval=5s --retries=12 --timeout=5s \
+    CMD curl --fail http://localhost:8000 || exit 1
 
-WORKDIR /code/api/restapi
-
-ENTRYPOINT ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
+FROM runtime as debug
+CMD ["uvicorn", "main:app", \
+    "--host", "0.0.0.0", "--port", "8000", \
+    "--reload", "--log-level", "critical", "--no-access-log"]
 
+FROM runtime as prod
+# Pre-compile packages to .pyc (init speed gains)
+RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
+# Note: 4 uvicorn workers as running with docker, change to 1 worker for Kubernetes
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", \
+    "--workers", "4", "--log-level", "critical", "--no-access-log"]