From 7d226d44f59be5200f734e6952079781ea5bbcfa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mart=C3=AD=20Bol=C3=ADvar?= <mbolivar@amperecomputing.com>
Date: Wed, 27 Sep 2023 12:51:15 -0700
Subject: [PATCH] Add docker compose based testing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Our release process documentation recommends getting passing tox
results on as many popular linux distributions as time allows. Doing
this by hand is cumbersome, redundant, and error prone.

Add a directory with a helper script that automates the entire process
using docker compose and document its use in MAINTAINERS.rst.

Signed-off-by: Martí Bolívar <mbolivar@amperecomputing.com>
Signed-off-by: Pieter De Gendt <pieter.degendt@basalte.be>
---
 .gitignore                               |  1 +
 MAINTAINERS.rst                          | 29 ++++++--
 docker-testing/README.rst                |  7 ++
 docker-testing/arch/Dockerfile           |  9 +++
 docker-testing/compose.yaml              | 88 ++++++++++++++++++++++++
 docker-testing/debian-stable/Dockerfile  | 10 +++
 docker-testing/debian-testing/Dockerfile | 10 +++
 docker-testing/fedora-latest/Dockerfile  |  9 +++
 docker-testing/fedora-rawhide/Dockerfile |  9 +++
 docker-testing/in-container-test.sh      | 39 +++++++++++
 docker-testing/run-tests.sh              | 14 ++++
 docker-testing/ubuntu-devel/Dockerfile   | 10 +++
 docker-testing/ubuntu-latest/Dockerfile  | 10 +++
 13 files changed, 238 insertions(+), 7 deletions(-)
 create mode 100644 docker-testing/README.rst
 create mode 100644 docker-testing/arch/Dockerfile
 create mode 100644 docker-testing/compose.yaml
 create mode 100644 docker-testing/debian-stable/Dockerfile
 create mode 100644 docker-testing/debian-testing/Dockerfile
 create mode 100644 docker-testing/fedora-latest/Dockerfile
 create mode 100644 docker-testing/fedora-rawhide/Dockerfile
 create mode 100755 docker-testing/in-container-test.sh
 create mode 100755 docker-testing/run-tests.sh
 create mode 100644 docker-testing/ubuntu-devel/Dockerfile
 create mode 100644 docker-testing/ubuntu-latest/Dockerfile

diff --git a/.gitignore b/.gitignore
index 2a9df372..bd34b8f8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ shippable/
 htmlcov/
 .dir-locals.el
 .venv/
+docker-testing/outdir
diff --git a/MAINTAINERS.rst b/MAINTAINERS.rst
index a79fce1a..47806dbf 100644
--- a/MAINTAINERS.rst
+++ b/MAINTAINERS.rst
@@ -11,20 +11,35 @@ Pre-release test plan
 
      git checkout vX.Y-branch
 
-1. Make tox happy on the following first-party platforms:
+1. Make tox happy on the following first-party non-Linux platforms:
 
    - Windows 10
    - the latest macOS
-   - the latest Ubuntu LTS
 
-2. Make tox happy on other popular Linux distributions as resources allow.
-   Doing this in a container is fine.
+   Do this by hand and check for any anomalous warnings in the output.
+   Do not just trust CI.
+
+2. Make tox happy on other popular Linux distributions:
 
    - Arch
-   - the latest Ubuntu release (if different than the latest LTS)
-   - Debian stable (if its Python 3 is still supported)
+   - the latest Ubuntu LTS release
+   - the latest Ubuntu development release
+   - Debian stable
    - Debian testing
-   - Fedora
+   - the latest Fedora release
+   - the latest Fedora rawhide release
+
+   Automated infrastructure for doing this in docker is in the docker-testing
+   directory. Start by updating the Dockerfiles and compose.yaml in that
+   directory if any newer distribution versions should be tested.
+
+   Then, install docker compose in your host Linux environment and run::
+
+     cd docker-testing
+     ./run-tests.sh
+
+   Make sure to check the tox.log files mentioned in the output for any
+   anomalous warnings.
 
 3. Build alpha N (N=1 to start, then N=2 if you need more commits, etc.) and
    upload to pypi. See "Building and uploading the release wheels" below for
diff --git a/docker-testing/README.rst b/docker-testing/README.rst
new file mode 100644
index 00000000..d2d1ec5f
--- /dev/null
+++ b/docker-testing/README.rst
@@ -0,0 +1,7 @@
+Docker based testing
+--------------------
+
+This directory contains helper files used for running west's tests in Docker on
+various Linux runtimes. It was originally developed for release testing.
+
+Run "./run-tests.sh" in this directory to run the tests.
diff --git a/docker-testing/arch/Dockerfile b/docker-testing/arch/Dockerfile
new file mode 100644
index 00000000..ef6121e6
--- /dev/null
+++ b/docker-testing/arch/Dockerfile
@@ -0,0 +1,9 @@
+FROM archlinux:latest
+CMD ["/west/docker-testing/in-container-test.sh"]
+
+RUN pacman -Syu --noconfirm \
+           git \
+           python-pip \
+    && pacman -Scc --noconfirm
+
+RUN pip3 install --break-system-packages tox
diff --git a/docker-testing/compose.yaml b/docker-testing/compose.yaml
new file mode 100644
index 00000000..fcadce27
--- /dev/null
+++ b/docker-testing/compose.yaml
@@ -0,0 +1,88 @@
+# It would be nicer to consolidate the common user/volumes
+# boilerplate, but there wasn't enough time to figure out if it was
+# possible at time of writing. Improvements welcome.
+
+services:
+  west-arch:
+    build: arch
+    environment:
+      WEST_TOX_OUT: /west/docker-testing/outdir/arch
+      WEST_TOX_OUT_IN_HOST: ${WEST_IN_HOST}/docker-testing/outdir/arch
+
+    user: ${MY_UID}:${MY_GID}
+    volumes:
+      - /etc/passwd:/etc/passwd:ro
+      - /etc/group:/etc/group:ro
+      - ..:/west
+
+  west-debian-stable:
+    build: debian-stable
+    environment:
+      WEST_TOX_OUT: /west/docker-testing/outdir/debian-stable
+      WEST_TOX_OUT_IN_HOST: ${WEST_IN_HOST}/docker-testing/outdir/debian-stable
+
+    user: ${MY_UID}:${MY_GID}
+    volumes:
+      - /etc/passwd:/etc/passwd:ro
+      - /etc/group:/etc/group:ro
+      - ..:/west
+
+  west-debian-testing:
+    build: debian-testing
+    environment:
+      WEST_TOX_OUT: /west/docker-testing/outdir/debian-testing
+      WEST_TOX_OUT_IN_HOST: ${WEST_IN_HOST}/docker-testing/outdir/debian-testing
+
+    user: ${MY_UID}:${MY_GID}
+    volumes:
+      - /etc/passwd:/etc/passwd:ro
+      - /etc/group:/etc/group:ro
+      - ..:/west
+
+  west-fedora-latest:
+    build: fedora-latest
+    environment:
+      WEST_TOX_OUT: /west/docker-testing/outdir/fedora-latest
+      WEST_TOX_OUT_IN_HOST: ${WEST_IN_HOST}/docker-testing/outdir/fedora-latest
+
+    user: ${MY_UID}:${MY_GID}
+    volumes:
+      - /etc/passwd:/etc/passwd:ro
+      - /etc/group:/etc/group:ro
+      - ..:/west
+
+  west-fedora-rawhide:
+    build: fedora-rawhide
+    environment:
+      WEST_TOX_OUT: /west/docker-testing/outdir/fedora-rawhide
+      WEST_TOX_OUT_IN_HOST: ${WEST_IN_HOST}/docker-testing/outdir/fedora-rawhide
+
+    user: ${MY_UID}:${MY_GID}
+    volumes:
+      - /etc/passwd:/etc/passwd:ro
+      - /etc/group:/etc/group:ro
+      - ..:/west
+
+  west-ubuntu-latest:
+    build: ubuntu-latest
+    environment:
+      WEST_TOX_OUT: /west/docker-testing/outdir/ubuntu-latest
+      WEST_TOX_OUT_IN_HOST: ${WEST_IN_HOST}/docker-testing/outdir/ubuntu-latest
+
+    user: ${MY_UID}:${MY_GID}
+    volumes:
+      - /etc/passwd:/etc/passwd:ro
+      - /etc/group:/etc/group:ro
+      - ..:/west
+
+  west-ubuntu-devel:
+    build: ubuntu-devel
+    environment:
+      WEST_TOX_OUT: /west/docker-testing/outdir/ubuntu-devel
+      WEST_TOX_OUT_IN_HOST: ${WEST_IN_HOST}/docker-testing/outdir/ubuntu-devel
+
+    user: ${MY_UID}:${MY_GID}
+    volumes:
+      - /etc/passwd:/etc/passwd:ro
+      - /etc/group:/etc/group:ro
+      - ..:/west
diff --git a/docker-testing/debian-stable/Dockerfile b/docker-testing/debian-stable/Dockerfile
new file mode 100644
index 00000000..5438ab7c
--- /dev/null
+++ b/docker-testing/debian-stable/Dockerfile
@@ -0,0 +1,10 @@
+FROM debian:stable
+CMD ["/west/docker-testing/in-container-test.sh"]
+
+RUN apt-get update \
+    && apt-get install -y \
+        git \
+        python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install --break-system-packages tox
diff --git a/docker-testing/debian-testing/Dockerfile b/docker-testing/debian-testing/Dockerfile
new file mode 100644
index 00000000..7ca504c7
--- /dev/null
+++ b/docker-testing/debian-testing/Dockerfile
@@ -0,0 +1,10 @@
+FROM debian:testing
+CMD ["/west/docker-testing/in-container-test.sh"]
+
+RUN apt-get update \
+    && apt-get install -y \
+        git \
+        python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install --break-system-packages tox
diff --git a/docker-testing/fedora-latest/Dockerfile b/docker-testing/fedora-latest/Dockerfile
new file mode 100644
index 00000000..22cc3d31
--- /dev/null
+++ b/docker-testing/fedora-latest/Dockerfile
@@ -0,0 +1,9 @@
+FROM fedora:latest
+CMD ["/west/docker-testing/in-container-test.sh"]
+
+RUN dnf install -y \
+        git \
+        python3-pip \
+    && dnf clean dbcache
+
+RUN pip3 install tox
diff --git a/docker-testing/fedora-rawhide/Dockerfile b/docker-testing/fedora-rawhide/Dockerfile
new file mode 100644
index 00000000..7d151d8d
--- /dev/null
+++ b/docker-testing/fedora-rawhide/Dockerfile
@@ -0,0 +1,9 @@
+FROM fedora:rawhide
+CMD ["/west/docker-testing/in-container-test.sh"]
+
+RUN dnf install -y \
+        git \
+        python3-pip \
+    && dnf clean dbcache
+
+RUN pip3 install tox
diff --git a/docker-testing/in-container-test.sh b/docker-testing/in-container-test.sh
new file mode 100755
index 00000000..6986bba3
--- /dev/null
+++ b/docker-testing/in-container-test.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+set -e
+
+# This is the test script that runs in the containers themselves.
+
+WEST=/west
+
+die() {
+    if [ $# -eq 0 ]; then
+        echo "error: $*" >&2
+    else
+        echo "error: unknown error in $0"
+    fi
+    exit 1
+}
+
+main()
+{
+    # Verify the container environment set up meets this script's requirements.
+    [ -n "$WEST_TOX_OUT" ] || die "missing $WEST_TOX_OUT"
+    [ -n "$WEST_TOX_OUT_IN_HOST" ] || die "missing $WEST_TOX_OUT_IN_HOST"
+    [ -d "$WEST" ] || die "missing $WEST in the container"
+
+    TOX_LOG="$WEST_TOX_OUT/tox.log"
+    TOX_LOG_IN_HOST="$WEST_TOX_OUT_IN_HOST/tox.log"
+    WEST_TESTDIR="/tmp/west"
+
+    mkdir "$WEST_TOX_OUT"
+
+    git clone -q "$WEST" "$WEST_TESTDIR" || die "failed to clone west to $WEST_TESTDIR in container"
+    cd "$WEST_TESTDIR"
+
+    echo "running tox, output in $TOX_LOG_IN_HOST in host"
+    tox run >"$TOX_LOG" 2>&1 || die "tox failed"
+
+    cp -R htmlcov "$WEST_TOX_OUT" || die "failed to copy coverage to $WEST_TOX_OUT_IN_HOST/htmlcov in host"
+}
+
+main "$@"
diff --git a/docker-testing/run-tests.sh b/docker-testing/run-tests.sh
new file mode 100755
index 00000000..de4b9b6b
--- /dev/null
+++ b/docker-testing/run-tests.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# This is the top-level test script that runs in the host.
+
+HERE=$(dirname "$0")
+
+[ -d "$HERE/outdir" ] && rm -r "$HERE/outdir"
+
+set -e
+mkdir "$HERE/outdir"
+export MY_UID=$(id -u)
+export MY_GID=$(id -g)
+export WEST_IN_HOST=$(realpath "$HERE/..")
+docker-compose up --force-recreate --build
diff --git a/docker-testing/ubuntu-devel/Dockerfile b/docker-testing/ubuntu-devel/Dockerfile
new file mode 100644
index 00000000..26456659
--- /dev/null
+++ b/docker-testing/ubuntu-devel/Dockerfile
@@ -0,0 +1,10 @@
+FROM ubuntu:devel
+CMD ["/west/docker-testing/in-container-test.sh"]
+
+RUN apt-get update \
+    && apt-get install -y \
+        git \
+        python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install --break-system-packages tox
diff --git a/docker-testing/ubuntu-latest/Dockerfile b/docker-testing/ubuntu-latest/Dockerfile
new file mode 100644
index 00000000..b24df54c
--- /dev/null
+++ b/docker-testing/ubuntu-latest/Dockerfile
@@ -0,0 +1,10 @@
+FROM ubuntu:latest
+CMD ["/west/docker-testing/in-container-test.sh"]
+
+RUN apt-get update \
+    && apt-get install -y \
+        git \
+        python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install --break-system-packages tox