From 25bf6c465c5b6082998cb9a9d3baa5203786b6e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=AD=20Bol=C3=ADvar?= Date: Wed, 27 Sep 2023 12:51:36 -0700 Subject: [PATCH 1/2] tox.ini: generate HTML coverage output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes it trivial to inspect, line by line, what got covered. That is useful when evaluating coverage for a new feature. Signed-off-by: Martí Bolívar --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 37089964..1685d884 100644 --- a/tox.ini +++ b/tox.ini @@ -36,6 +36,6 @@ deps = setenv = TOXTEMPDIR={envtmpdir} commands = - python -m pytest --cov=west {posargs:tests} + python -m pytest --cov-report=html --cov=west {posargs:tests} python -m flake8 --config={toxinidir}/tox.ini {toxinidir} python -m mypy --config-file={toxinidir}/tox.ini --package=west From 8049708c27da0306678d8667ddf829820c306d2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=AD=20Bol=C3=ADvar?= Date: Wed, 27 Sep 2023 12:51:15 -0700 Subject: [PATCH 2/2] Add docker compose based testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Our release process documentation recommends getting passing tox results on as many popular linux distributions as time allows. Doing this by hand is cumbersome, redundant, and error prone. Add a directory with a helper script that automates the entire process using docker compose and document its use in MAINTAINERS.rst. Signed-off-by: Martí Bolívar --- .gitignore | 1 + MAINTAINERS.rst | 28 ++++++--- docker-testing/README.rst | 7 +++ docker-testing/arch/Dockerfile | 9 +++ docker-testing/compose.yaml | 76 ++++++++++++++++++++++++ docker-testing/debian-12/Dockerfile | 10 ++++ docker-testing/debian-testing/Dockerfile | 10 ++++ docker-testing/fedora-38/Dockerfile | 9 +++ docker-testing/in-container-test.sh | 38 ++++++++++++ docker-testing/run-tests.sh | 14 +++++ docker-testing/ubuntu-22.04/Dockerfile | 10 ++++ docker-testing/ubuntu-23.04/Dockerfile | 10 ++++ 12 files changed, 215 insertions(+), 7 deletions(-) create mode 100644 docker-testing/README.rst create mode 100644 docker-testing/arch/Dockerfile create mode 100644 docker-testing/compose.yaml create mode 100644 docker-testing/debian-12/Dockerfile create mode 100644 docker-testing/debian-testing/Dockerfile create mode 100644 docker-testing/fedora-38/Dockerfile create mode 100755 docker-testing/in-container-test.sh create mode 100755 docker-testing/run-tests.sh create mode 100644 docker-testing/ubuntu-22.04/Dockerfile create mode 100644 docker-testing/ubuntu-23.04/Dockerfile diff --git a/.gitignore b/.gitignore index 2e73ec42..1eff6e71 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ shippable/ # htmlcov is generated by tox due to coverage gathering in pytest htmlcov/ .dir-locals.el +docker-testing/outdir diff --git a/MAINTAINERS.rst b/MAINTAINERS.rst index a79fce1a..6613524c 100644 --- a/MAINTAINERS.rst +++ b/MAINTAINERS.rst @@ -11,20 +11,34 @@ Pre-release test plan git checkout vX.Y-branch -1. Make tox happy on the following first-party platforms: +1. Make tox happy on the following first-party non-Linux platforms: - Windows 10 - the latest macOS - - the latest Ubuntu LTS -2. Make tox happy on other popular Linux distributions as resources allow. - Doing this in a container is fine. + Do this by hand and check for any anomalous warnings in the output. + Do not just trust CI. + +2. Make tox happy on other popular Linux distributions: - Arch - - the latest Ubuntu release (if different than the latest LTS) - - Debian stable (if its Python 3 is still supported) + - the latest Ubuntu LTS (22.04 at time of writing) + - the latest Ubuntu release (23.04 at time of writing) + - Debian stable (12 at time of writing) - Debian testing - - Fedora + - the latest Fedora (38 at time of writing) + + Automated infrastructure for doing this in docker is in the docker-testing + directory. Start by updating the Dockerfiles and compose.yaml in that + directory if any newer distribution versions should be tested. + + Then, install docker compose in your host Linux environment and run:: + + cd docker-testing + ./run-tests.sh + + Make sure to check the tox.log files mentioned in the output for any + anomalous warnings. 3. Build alpha N (N=1 to start, then N=2 if you need more commits, etc.) and upload to pypi. See "Building and uploading the release wheels" below for diff --git a/docker-testing/README.rst b/docker-testing/README.rst new file mode 100644 index 00000000..d2d1ec5f --- /dev/null +++ b/docker-testing/README.rst @@ -0,0 +1,7 @@ +Docker based testing +-------------------- + +This directory contains helper files used for running west's tests in Docker on +various Linux runtimes. It was originally developed for release testing. + +Run "./run-tests.sh" in this directory to run the tests. diff --git a/docker-testing/arch/Dockerfile b/docker-testing/arch/Dockerfile new file mode 100644 index 00000000..ef6121e6 --- /dev/null +++ b/docker-testing/arch/Dockerfile @@ -0,0 +1,9 @@ +FROM archlinux:latest +CMD ["/west/docker-testing/in-container-test.sh"] + +RUN pacman -Syu --noconfirm \ + git \ + python-pip \ + && pacman -Scc --noconfirm + +RUN pip3 install --break-system-packages tox diff --git a/docker-testing/compose.yaml b/docker-testing/compose.yaml new file mode 100644 index 00000000..217b8604 --- /dev/null +++ b/docker-testing/compose.yaml @@ -0,0 +1,76 @@ +# It would be nicer to consolidate the common user/volumes +# boilerplate, but there wasn't enough time to figure out if it was +# possible at time of writing. Improvements welcome. + +services: + west-arch: + build: arch + environment: + WEST_TOX_OUT: /west/docker-testing/outdir/arch + WEST_TOX_OUT_IN_HOST: ${WEST_IN_HOST}/docker-testing/outdir/arch + + user: ${MY_UID}:${MY_GID} + volumes: + - /etc/passwd:/etc/passwd:ro + - /etc/group:/etc/group:ro + - ..:/west + + west-debian-12: + build: debian-12 + environment: + WEST_TOX_OUT: /west/docker-testing/outdir/debian-12 + WEST_TOX_OUT_IN_HOST: ${WEST_IN_HOST}/docker-testing/outdir/debian-12 + + user: ${MY_UID}:${MY_GID} + volumes: + - /etc/passwd:/etc/passwd:ro + - /etc/group:/etc/group:ro + - ..:/west + + west-debian-testing: + build: debian-testing + environment: + WEST_TOX_OUT: /west/docker-testing/outdir/debian-testing + WEST_TOX_OUT_IN_HOST: ${WEST_IN_HOST}/docker-testing/outdir/debian-testing + + user: ${MY_UID}:${MY_GID} + volumes: + - /etc/passwd:/etc/passwd:ro + - /etc/group:/etc/group:ro + - ..:/west + + west-fedora-38: + build: fedora-38 + environment: + WEST_TOX_OUT: /west/docker-testing/outdir/fedora-38 + WEST_TOX_OUT_IN_HOST: ${WEST_IN_HOST}/docker-testing/outdir/fedora-38 + + user: ${MY_UID}:${MY_GID} + volumes: + - /etc/passwd:/etc/passwd:ro + - /etc/group:/etc/group:ro + - ..:/west + + west-ubuntu-22.04: + build: ubuntu-22.04 + environment: + WEST_TOX_OUT: /west/docker-testing/outdir/ubuntu-22.04 + WEST_TOX_OUT_IN_HOST: ${WEST_IN_HOST}/docker-testing/outdir/ubuntu-22.04 + + user: ${MY_UID}:${MY_GID} + volumes: + - /etc/passwd:/etc/passwd:ro + - /etc/group:/etc/group:ro + - ..:/west + + west-ubuntu-23.04: + build: ubuntu-23.04 + environment: + WEST_TOX_OUT: /west/docker-testing/outdir/ubuntu-23.04 + WEST_TOX_OUT_IN_HOST: ${WEST_IN_HOST}/docker-testing/outdir/ubuntu-23.04 + + user: ${MY_UID}:${MY_GID} + volumes: + - /etc/passwd:/etc/passwd:ro + - /etc/group:/etc/group:ro + - ..:/west diff --git a/docker-testing/debian-12/Dockerfile b/docker-testing/debian-12/Dockerfile new file mode 100644 index 00000000..d66b1e12 --- /dev/null +++ b/docker-testing/debian-12/Dockerfile @@ -0,0 +1,10 @@ +FROM debian:bookworm +CMD ["/west/docker-testing/in-container-test.sh"] + +RUN apt-get update \ + && apt-get install -y \ + git \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install --break-system-packages tox diff --git a/docker-testing/debian-testing/Dockerfile b/docker-testing/debian-testing/Dockerfile new file mode 100644 index 00000000..7ca504c7 --- /dev/null +++ b/docker-testing/debian-testing/Dockerfile @@ -0,0 +1,10 @@ +FROM debian:testing +CMD ["/west/docker-testing/in-container-test.sh"] + +RUN apt-get update \ + && apt-get install -y \ + git \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install --break-system-packages tox diff --git a/docker-testing/fedora-38/Dockerfile b/docker-testing/fedora-38/Dockerfile new file mode 100644 index 00000000..137657af --- /dev/null +++ b/docker-testing/fedora-38/Dockerfile @@ -0,0 +1,9 @@ +FROM fedora:38 +CMD ["/west/docker-testing/in-container-test.sh"] + +RUN dnf install -y \ + git \ + python3-pip \ + && dnf clean dbcache + +RUN pip3 install tox diff --git a/docker-testing/in-container-test.sh b/docker-testing/in-container-test.sh new file mode 100755 index 00000000..bf329009 --- /dev/null +++ b/docker-testing/in-container-test.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# This is the test script that runs in the containers themselves. + +WEST=/west + +die() { + if [ ! -z "$@" ]; then + echo "error: $@" >&2 + else + echo "error: unknown error in $0" + fi + exit 1 +} + +main() +{ + # Verify the container environment set up meets this script's requirements. + [ ! -z "$WEST_TOX_OUT" ] || die "missing $WEST_TOX_OUT" + [ ! -z "$WEST_TOX_OUT_IN_HOST" ] || die "missing $WEST_TOX_OUT_IN_HOST" + [ -d "$WEST" ] || die "missing $WEST in the container" + + TOX_LOG="$WEST_TOX_OUT/tox.log" + TOX_LOG_IN_HOST="$WEST_TOX_OUT_IN_HOST/tox.log" + WEST_TESTDIR="/tmp/west" + + mkdir "$WEST_TOX_OUT" || die "failed to make $WEST_TOX_OUT in container ($WEST_TOX_OUT_IN_HOST in host)" + + git clone -q "$WEST" "$WEST_TESTDIR" || die "failed to clone west to $WEST_TESTDIR in container" + cd "$WEST_TESTDIR" + + echo "running tox, output in $TOX_LOG_IN_HOST in host" + tox run >"$TOX_LOG" 2>&1 || die "tox failed" + + cp -R htmlcov "$WEST_TOX_OUT" || die "failed to copy coverage to $WEST_TOX_OUT_IN_HOST/htmlcov in host" +} + +main "$@" diff --git a/docker-testing/run-tests.sh b/docker-testing/run-tests.sh new file mode 100755 index 00000000..de4b9b6b --- /dev/null +++ b/docker-testing/run-tests.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# This is the top-level test script that runs in the host. + +HERE=$(dirname "$0") + +[ -d "$HERE/outdir" ] && rm -r "$HERE/outdir" + +set -e +mkdir "$HERE/outdir" +export MY_UID=$(id -u) +export MY_GID=$(id -g) +export WEST_IN_HOST=$(realpath "$HERE/..") +docker-compose up --force-recreate --build diff --git a/docker-testing/ubuntu-22.04/Dockerfile b/docker-testing/ubuntu-22.04/Dockerfile new file mode 100644 index 00000000..921d6dc4 --- /dev/null +++ b/docker-testing/ubuntu-22.04/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:22.04 +CMD ["/west/docker-testing/in-container-test.sh"] + +RUN apt-get update \ + && apt-get install -y \ + git \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install tox diff --git a/docker-testing/ubuntu-23.04/Dockerfile b/docker-testing/ubuntu-23.04/Dockerfile new file mode 100644 index 00000000..a7dc8dcb --- /dev/null +++ b/docker-testing/ubuntu-23.04/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:23.04 +CMD ["/west/docker-testing/in-container-test.sh"] + +RUN apt-get update \ + && apt-get install -y \ + git \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install --break-system-packages tox