From 83f123f4485042fa0848a3bb5c0a6e73632e8d73 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 24 Nov 2023 16:45:38 +0000 Subject: [PATCH] Switch to python-tools-scripts Signed-off-by: Pedro Algarvio --- .github/workflows/.container.template.j2 | 40 +-- .github/workflows/almalinux-containers.yml | 40 +-- .github/workflows/amazonlinux-containers.yml | 40 +-- .github/workflows/archlinux-containers.yml | 40 +-- .github/workflows/busybox-containers.yml | 40 +-- .github/workflows/centos-containers.yml | 40 +-- .../workflows/centos-stream-containers.yml | 40 +-- .github/workflows/consul-containers.yml | 40 +-- .github/workflows/debian-containers.yml | 40 +-- .github/workflows/etcd-v2-containers.yml | 40 +-- .github/workflows/etcd-v3-containers.yml | 40 +-- .github/workflows/fedora-containers.yml | 40 +-- .github/workflows/mariadb-containers.yml | 40 +-- .github/workflows/mysql-server-containers.yml | 40 +-- .../workflows/openldap-minion-containers.yml | 40 +-- .github/workflows/opensuse-containers.yml | 40 +-- .github/workflows/packaging-containers.yml | 40 +-- .github/workflows/percona-containers.yml | 40 +-- .github/workflows/photon-containers.yml | 40 +-- .github/workflows/python-containers.yml | 40 +-- .github/workflows/rabbitmq-containers.yml | 40 +-- .github/workflows/redis-containers.yml | 40 +-- .github/workflows/salt-containers.yml | 40 +-- .../workflows/salt-gitfs-http-containers.yml | 40 +-- .github/workflows/ssh-minion-containers.yml | 40 +-- .github/workflows/tinyproxy-containers.yml | 40 +-- .github/workflows/ubuntu-containers.yml | 40 +-- .github/workflows/vault-containers.yml | 40 +-- .github/workflows/virt-minion-containers.yml | 40 +-- .github/workflows/zookeeper-containers.yml | 40 +-- .pre-commit-config.yaml | 39 +-- requirements.in | 3 +- requirements.txt | 30 +- tasks/__init__.py | 8 - tasks/containers.py | 316 ------------------ tasks/utils.py | 80 ----- tools/__init__.py | 11 + tools/ci.py | 144 ++++++++ tools/pre_commit.py | 167 +++++++++ tools/utils.py | 67 ++++ 40 files changed, 969 insertions(+), 1096 deletions(-) delete mode 100644 tasks/__init__.py delete mode 100644 tasks/containers.py delete mode 100644 tasks/utils.py create mode 100644 tools/ci.py create mode 100644 tools/utils.py diff --git a/.github/workflows/.container.template.j2 b/.github/workflows/.container.template.j2 index 1960ebb..7b90606 100644 --- a/.github/workflows/.container.template.j2 +++ b/.github/workflows/.container.template.j2 @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/almalinux-containers.yml b/.github/workflows/almalinux-containers.yml index 7276a84..c13505a 100644 --- a/.github/workflows/almalinux-containers.yml +++ b/.github/workflows/almalinux-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/amazonlinux-containers.yml b/.github/workflows/amazonlinux-containers.yml index a703254..9784d15 100644 --- a/.github/workflows/amazonlinux-containers.yml +++ b/.github/workflows/amazonlinux-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/archlinux-containers.yml b/.github/workflows/archlinux-containers.yml index 2a00df8..aca14e2 100644 --- a/.github/workflows/archlinux-containers.yml +++ b/.github/workflows/archlinux-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/busybox-containers.yml b/.github/workflows/busybox-containers.yml index 790ff60..c2b8348 100644 --- a/.github/workflows/busybox-containers.yml +++ b/.github/workflows/busybox-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/centos-containers.yml b/.github/workflows/centos-containers.yml index 0e4f7ea..2413663 100644 --- a/.github/workflows/centos-containers.yml +++ b/.github/workflows/centos-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/centos-stream-containers.yml b/.github/workflows/centos-stream-containers.yml index 4eca397..c31e6aa 100644 --- a/.github/workflows/centos-stream-containers.yml +++ b/.github/workflows/centos-stream-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/consul-containers.yml b/.github/workflows/consul-containers.yml index 24ef688..4e098d9 100644 --- a/.github/workflows/consul-containers.yml +++ b/.github/workflows/consul-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/debian-containers.yml b/.github/workflows/debian-containers.yml index 9ebc9d4..619595a 100644 --- a/.github/workflows/debian-containers.yml +++ b/.github/workflows/debian-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/etcd-v2-containers.yml b/.github/workflows/etcd-v2-containers.yml index dd3a3b0..c3a9e2b 100644 --- a/.github/workflows/etcd-v2-containers.yml +++ b/.github/workflows/etcd-v2-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/etcd-v3-containers.yml b/.github/workflows/etcd-v3-containers.yml index 83c481a..49df1be 100644 --- a/.github/workflows/etcd-v3-containers.yml +++ b/.github/workflows/etcd-v3-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/fedora-containers.yml b/.github/workflows/fedora-containers.yml index f8b10dc..af0e3d4 100644 --- a/.github/workflows/fedora-containers.yml +++ b/.github/workflows/fedora-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/mariadb-containers.yml b/.github/workflows/mariadb-containers.yml index f06d082..da9ff91 100644 --- a/.github/workflows/mariadb-containers.yml +++ b/.github/workflows/mariadb-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/mysql-server-containers.yml b/.github/workflows/mysql-server-containers.yml index bf84186..0cc9e09 100644 --- a/.github/workflows/mysql-server-containers.yml +++ b/.github/workflows/mysql-server-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/openldap-minion-containers.yml b/.github/workflows/openldap-minion-containers.yml index 84c1900..fba3c00 100644 --- a/.github/workflows/openldap-minion-containers.yml +++ b/.github/workflows/openldap-minion-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/opensuse-containers.yml b/.github/workflows/opensuse-containers.yml index cdf3241..6961bc5 100644 --- a/.github/workflows/opensuse-containers.yml +++ b/.github/workflows/opensuse-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/packaging-containers.yml b/.github/workflows/packaging-containers.yml index 77a4736..4d80201 100644 --- a/.github/workflows/packaging-containers.yml +++ b/.github/workflows/packaging-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/percona-containers.yml b/.github/workflows/percona-containers.yml index c29e849..21e156a 100644 --- a/.github/workflows/percona-containers.yml +++ b/.github/workflows/percona-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/photon-containers.yml b/.github/workflows/photon-containers.yml index 59c2d9a..866cfb9 100644 --- a/.github/workflows/photon-containers.yml +++ b/.github/workflows/photon-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/python-containers.yml b/.github/workflows/python-containers.yml index ced7e1e..aeee45d 100644 --- a/.github/workflows/python-containers.yml +++ b/.github/workflows/python-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/rabbitmq-containers.yml b/.github/workflows/rabbitmq-containers.yml index fee728a..4e1c12a 100644 --- a/.github/workflows/rabbitmq-containers.yml +++ b/.github/workflows/rabbitmq-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/redis-containers.yml b/.github/workflows/redis-containers.yml index c53ee0d..cf99027 100644 --- a/.github/workflows/redis-containers.yml +++ b/.github/workflows/redis-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/salt-containers.yml b/.github/workflows/salt-containers.yml index 0f6b057..363b94c 100644 --- a/.github/workflows/salt-containers.yml +++ b/.github/workflows/salt-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/salt-gitfs-http-containers.yml b/.github/workflows/salt-gitfs-http-containers.yml index 10fca9d..98afe76 100644 --- a/.github/workflows/salt-gitfs-http-containers.yml +++ b/.github/workflows/salt-gitfs-http-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/ssh-minion-containers.yml b/.github/workflows/ssh-minion-containers.yml index 3512108..99bd32c 100644 --- a/.github/workflows/ssh-minion-containers.yml +++ b/.github/workflows/ssh-minion-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/tinyproxy-containers.yml b/.github/workflows/tinyproxy-containers.yml index e8e0e30..1668c28 100644 --- a/.github/workflows/tinyproxy-containers.yml +++ b/.github/workflows/tinyproxy-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/ubuntu-containers.yml b/.github/workflows/ubuntu-containers.yml index bbba123..b9e28da 100644 --- a/.github/workflows/ubuntu-containers.yml +++ b/.github/workflows/ubuntu-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/vault-containers.yml b/.github/workflows/vault-containers.yml index 6530e33..ccab71d 100644 --- a/.github/workflows/vault-containers.yml +++ b/.github/workflows/vault-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/virt-minion-containers.yml b/.github/workflows/virt-minion-containers.yml index 4714159..a49a664 100644 --- a/.github/workflows/virt-minion-containers.yml +++ b/.github/workflows/virt-minion-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.github/workflows/zookeeper-containers.yml b/.github/workflows/zookeeper-containers.yml index 02c0dd8..5257713 100644 --- a/.github/workflows/zookeeper-containers.yml +++ b/.github/workflows/zookeeper-containers.yml @@ -45,23 +45,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' - name: Install Requirements run: | python -m pip install -r requirements.txt - - name: "Docker QEMU" - uses: docker/setup-qemu-action@v3 - - - name: "Docker BuildX" - id: buildx - uses: docker/setup-buildx-action@v3 - - name: "Generate Matrix Data" id: set-matrix run: | - inv containers.matrix --from-workflow ${{ env.PATH_IN_REPO }} + tools ci matrix ${{ env.PATH_IN_REPO }} build: runs-on: ubuntu-latest @@ -86,15 +79,6 @@ jobs: - name: "Fetching Repository Contents" uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Requirements - run: | - python -m pip install -r requirements.txt - - name: "Throttle concurrent pushes" run: | t="$(shuf -i 5-30 -n 1)"; echo "Sleeping $t seconds"; sleep "$t" @@ -147,7 +131,7 @@ jobs: if: ${{ github.repository == 'saltstack/salt-ci-containers' && contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }} run: | mkdir -p /tmp/digests - touch /tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ":" -f 2) + touch "/tmp/digests/$(echo ${{ steps.build.outputs.digest }} | cut -d ':' -f 2)" ls -lah /tmp/digests - name: Upload digest @@ -184,12 +168,25 @@ jobs: statuses: read steps: + + - name: "Fetching Repository Contents" + uses: actions/checkout@v4 + - name: Download digests uses: actions/download-artifact@v3 with: name: digests-${{ needs.matrix-generator.outputs.name }}-${{ matrix.tag }} path: /tmp/digests + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Requirements + run: | + python -m pip install -r requirements.txt + - name: Show digests run: | tree -a /tmp/digests @@ -218,9 +215,8 @@ jobs: - name: Create manifest list and push working-directory: /tmp/digests - run: | # shellcheck disable=SC2046 - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf 'ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }}@sha256:%s ' *) + run: + tools ci create-manifest-list-and-push ghcr.io/${{ github.repository }}/${{ needs.matrix-generator.outputs.name }} /tmp/digests - name: Inspect image run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec1ebc2..30de4eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,11 +2,21 @@ minimum_pre_commit_version: 2.4.0 repos: - - repo: https://github.com/saltstack/invoke-pre-commit - rev: v1.9.0 + - repo: https://github.com/s0undt3ch/python-tools-scripts + rev: "0.18.4" hooks: - - id: invoke - alias: generate-containers + - id: tools + alias: actionlint + name: Lint GitHub Actions Workflows + files: "^.github/workflows/" + types: + - yaml + args: + - pre-commit + - actionlint + + - id: tools + alias: generate-workflows name: Generate Containers files: > (?x)^( @@ -19,28 +29,9 @@ repos: .*\.Dockerfile )$ pass_filenames: false - args: - - containers.generate - additional_dependencies: - - blessings==1.7 - - pyyaml==6.0 - - markupsafe==2.1.1 - - jinja2==3.1.2 - - msgpack==1.0.3 - - six==1.16.0 - - - repo: https://github.com/s0undt3ch/python-tools-scripts - rev: "0.18.4" - hooks: - - id: tools - alias: actionlint - name: Lint GitHub Actions Workflows - files: "^.github/workflows/" - types: - - yaml args: - pre-commit - - actionlint + - generate-workflows - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 diff --git a/requirements.in b/requirements.in index b8a77c7..82ddd27 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,3 @@ -invoke -blessings pyyaml jinja2 +python-tools-scripts diff --git a/requirements.txt b/requirements.txt index a3f0743..2ccb057 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,15 +4,33 @@ # # pip-compile --output-file=requirements.txt requirements.in # -blessings==1.7 - # via -r requirements.in -invoke==1.7.1 - # via -r requirements.in +attrs==23.1.0 + # via python-tools-scripts +certifi==2023.11.17 + # via requests +charset-normalizer==3.3.2 + # via requests +idna==3.4 + # via requests jinja2==3.1.2 # via -r requirements.in +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.1 # via jinja2 +mdurl==0.1.2 + # via markdown-it-py +pygments==2.17.2 + # via rich +python-tools-scripts==0.18.4 + # via -r requirements.in pyyaml==6.0 # via -r requirements.in -six==1.16.0 - # via blessings +requests==2.31.0 + # via python-tools-scripts +rich==13.7.0 + # via python-tools-scripts +typing-extensions==4.8.0 + # via python-tools-scripts +urllib3==2.1.0 + # via requests diff --git a/tasks/__init__.py b/tasks/__init__.py deleted file mode 100644 index cf97047..0000000 --- a/tasks/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import annotations - -from invoke import Collection - -from . import containers - -ns = Collection() -ns.add_collection(Collection.from_module(containers, name="containers"), name="containers") diff --git a/tasks/containers.py b/tasks/containers.py deleted file mode 100644 index f2eb6d9..0000000 --- a/tasks/containers.py +++ /dev/null @@ -1,316 +0,0 @@ -""" -Generate the mirrors layout. -""" -from __future__ import annotations - -import json -import os -import pprint - -import jinja2.sandbox -import yaml -from invoke import task - -from . import utils - - -@task -def generate(ctx, ghcr_org="saltstack/salt-ci-containers"): - """ - Generate the container mirrors. - """ - ctx.cd(utils.REPO_ROOT) - containers = _get_containers() - - main_readme = utils.REPO_ROOT / "README.md" - main_readme_contents = [] - - for line in main_readme.read_text().splitlines(): - if line == "": - main_readme_contents.append(line) - main_readme_contents.append("\n## Salt Releases") - break - else: - main_readme_contents.append(line) - - ctx.run( - "git rm -f mirrors/*/*.Dockerfile .github/workflows/*-containers.yml", warn=True, hide=True - ) - for path in utils.REPO_ROOT.joinpath("mirrors").glob("*"): - if list(path.glob("*")) == [path / "README.md"]: - ctx.run(f"git rm -rf {path}", warn=True, hide=True) - - custom_headers_included = False - mirrors_header_included = False - cron_hour_range = list(range(0, 24)) - for name, details in containers: - is_mirror = details["is_mirror"] - - if is_mirror: - if not mirrors_header_included: - main_readme_contents.append("\n\n## Mirrors") - mirrors_header_included = True - - utils.info(f"Generating {name} mirror...") - container = details["container"] - if "/" in container: - org, container_name = container.rsplit("/", 1) - else: - org = "_" - container_name = container - - source_tag = details.get("source_tag") - container_dir = utils.REPO_ROOT / "mirrors" / (details.get("dest") or details["name"]) - container_dir.mkdir(parents=True, exist_ok=True) - else: - org = ghcr_org - container_name = details["name"] - if details["name"] == "salt": - container_dir = utils.REPO_ROOT / container_name - else: - if not custom_headers_included: - main_readme_contents.append("\n## Custom") - custom_headers_included = True - container_dir = utils.REPO_ROOT / "custom" / container_name - - readme = container_dir / "README.md" - readme_contents = [] - for version in sorted(details["versions"]): - utils.info(f" Generating docker file for version {version}...") - dockerfile = container_dir / f"{version}.Dockerfile" - if is_mirror: - local_source_tag = source_tag - if local_source_tag is not None: - local_source_tag = local_source_tag.format(version=version) - dockerfile_exists = dockerfile.exists() - if container.count("/") > 1: - hosting = f"https://{org}/{container_name}" - if "quay" in org: - hosting += f"?tab=tags&tag={local_source_tag or version}" - else: - hosting = f"https://hub.docker.com/r/{org}/{container_name}/tags?name={local_source_tag or version}" - readme_contents.append( - f"- [{org if org != '_' else 'dockerhub'}/{container_name}:{local_source_tag or version}]" - f"({hosting}) - `ghcr.io/{ghcr_org}/{details['name']}:{version}`" - ) - source_container = f"{container}:{local_source_tag or version}" - with dockerfile.open("w") as wfh: - wfh.write(f"FROM {source_container}\n") - for command in details.get("commands", ()): - wfh.write(f"RUN {command}\n") - if not dockerfile_exists: - ctx.run(f"git add {dockerfile.relative_to(utils.REPO_ROOT)}") - else: - source_container = _get_source_container(dockerfile) - readme_contents.append( - f"- {container_name}:{version} - `ghcr.io/{ghcr_org}/{container_name}:{version}`" - ) - - readme_exists = readme.exists() - workflow_file_name = f"{details.get('dest') or details['name']}-containers.yml" - with readme.open("w") as wfh: - header = ( - f"# [![{name}]" - f"(https://github.com/{ghcr_org}/actions/workflows/{workflow_file_name}/badge.svg)]" - f"(https://github.com/{ghcr_org}/actions/workflows/{workflow_file_name})\n" - ) - main_readme_contents.append("\n") - main_readme_contents.append(f"##{header}") - main_readme_contents.extend(readme_contents) - wfh.write(f"{header}\n") - wfh.write("\n".join(readme_contents)) - wfh.write("\n") - - if not readme_exists: - ctx.run(f"git add {readme.relative_to(utils.REPO_ROOT)}") - - utils.info(f" Generating Github workflow for {name} mirror...") - env = jinja2.sandbox.SandboxedEnvironment( - block_start_string="<%", - block_end_string="%>", - variable_start_string="<{", - variable_end_string="}>", - extensions=[ - "jinja2.ext.do", - ], - ) - workflow_tpl = utils.REPO_ROOT / ".github" / "workflows" / ".container.template.j2" - template = env.from_string(workflow_tpl.read_text()) - cron_hour = cron_hour_range.pop() - if not cron_hour_range: - cron_hour_range = list(range(0, 24)) - jinja_context = { - "name": name, - "slug": details["name"], - "repository_path": container_dir.relative_to(utils.REPO_ROOT), - "is_mirror": is_mirror, - "workflow_file_name": workflow_file_name, - "cron_hour": cron_hour, - } - workflows_dir = utils.REPO_ROOT / ".github" / "workflows" - workflow_path = workflows_dir / workflow_file_name - workflow_path_exists = workflow_path.exists() - utils.info(f" Writing {workflow_path.relative_to(utils.REPO_ROOT)} ...") - workflow_path.write_text(template.render(**jinja_context).rstrip() + "\n") - if not workflow_path_exists: - ctx.run(f"git add {workflow_path.relative_to(utils.REPO_ROOT)}") - - main_readme_contents[-1] = main_readme_contents[-1].rstrip() - main_readme_contents.append("\n") - - with main_readme.open("w") as wfh: - contents = "\n".join(main_readme_contents).rstrip() - wfh.write(f"{contents}\n") - - ctx.run("git add .github/workflows/*-containers.yml") - - -@task -def matrix(ctx, image, from_workflow=False, build_platforms=None): - """ - Generate the container mirrors. - """ - ctx.cd(utils.REPO_ROOT) - mirrors_path = utils.REPO_ROOT / image - - if build_platforms is None: - build_platforms = ["linux/amd64", "linux/arm64/v8"] - else: - build_platforms = build_platforms.split(",") - - for name, details in _get_containers(): - if details["path"] == image: - break - else: - utils.error(f"Failed to find a container matching path {image}") - utils.exit_invoke(1) - output = [] - tags = [] - for fpath in mirrors_path.glob("*.Dockerfile"): - tags.append(fpath.stem) - if "container" in details: - source_tag = details.get("source_tag", "{version}").format(version=fpath.stem) - source_container = f"{details['container']}:{source_tag}" - else: - source_container = _get_source_container(fpath) - - ret = ctx.run( - f"docker buildx imagetools inspect --raw {source_container}", echo=False, hide=True - ) - data = json.loads(ret.stdout) - for entry in data.get("manifests", ()): - platform = "{os}/{architecture}".format(**entry["platform"]) - if "variant" in entry["platform"]: - platform += f"/{entry['platform']['variant']}" - if platform not in build_platforms: - continue - output.append( - { - "name": details["name"], - "tag": fpath.stem, - "platform": platform, - "file": str(fpath.relative_to(utils.REPO_ROOT)), - "source_container": source_container, - } - ) - - if not output: - # This is because the buildx inspect did not return anything - output.append( - { - "name": details["name"], - "tag": fpath.stem, - "file": str(fpath.relative_to(utils.REPO_ROOT)), - "source_container": source_container, - "platform": "", - } - ) - - utils.info("Generated Matrix:") - utils.write_message(pprint.pformat(output)) - - if from_workflow: - github_output = os.environ.get("GITHUB_OUTPUT") - if github_output is None: - utils.warn("The 'GITHUB_OUTPUT' variable is not set.") - utils.exit_invoke(1) - with open(github_output, "a", encoding="utf-8") as wfh: - wfh.write(f"dockerinfo={json.dumps(output)}\n") - wfh.write(f"tags={json.dumps(tags)}\n") - wfh.write(f"name={details['name']}") - - -@task -def platforms(ctx, supported_platforms_file, exclude=None): - """ - Generate a platforms list. - """ - if exclude is None: - excludes = [] - else: - excludes = exclude.split(",") - platforms = {"linux/amd64"} - with open(supported_platforms_file, encoding="utf-8") as rfh: - for line in rfh: - if "Platform:" in line: - _, platform = line.split() - if platform in excludes: - utils.info("Excluding {}", platform) - continue - utils.info("Inluding {}", platform) - platforms.add(platform) - - contents = "platforms={}".format(",".join(sorted(platforms))) - utils.info("Writing '{}' to $GITHUB_OUTPUT ...", contents) - with open(os.environ["GITHUB_OUTPUT"], "w", encoding="utf-8") as wfh: - wfh.write(f"{contents}\n") - - -def _get_containers(): - containers_path = utils.REPO_ROOT / "containers.yml" - if containers_path.exists(): - with containers_path.open("r") as rfh: - loaded_containers = yaml.safe_load(rfh.read()) - else: - loaded_containers = { - "salt": {}, - "custom": {}, - "mirrors": {}, - } - - salt_containers = loaded_containers["salt"] - custom_containers = loaded_containers["custom"] - mirror_containers = loaded_containers["mirrors"] - for name, details in salt_containers.items(): - details["is_mirror"] = False - details["path"] = "salt" - salt_containers[name] = details - - for name, details in custom_containers.items(): - details["is_mirror"] = False - details["path"] = os.path.join("custom", details.get("dest") or details["name"]) - custom_containers[name] = details - - for name, details in mirror_containers.items(): - details["is_mirror"] = True - if "name" not in details: - details["name"] = name.lower().replace(" ", "-") - details["path"] = os.path.join("mirrors", details.get("dest") or details["name"]) - mirror_containers[name] = details - - return list(sorted(salt_containers.items()) + sorted(custom_containers.items())) + list( - sorted(mirror_containers.items()) - ) - - -def _get_source_container(dockerfile): - source_container = None - with dockerfile.open("r") as rfh: - for line in rfh: - if line.startswith("FROM "): - source_container = line.strip().split()[-1] - break - else: - utils.error("Failed to find 'FROM ' line") - utils.exit_invoke(1) - return source_container diff --git a/tasks/utils.py b/tasks/utils.py deleted file mode 100644 index 60acee9..0000000 --- a/tasks/utils.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -Invoke utilities. -""" -from __future__ import annotations - -import pathlib -import sys - -try: - from blessings import Terminal - - try: - terminal = Terminal() - HAS_BLESSINGS = True - except Exception: # pylint: disable=broad-except - terminal = None - HAS_BLESSINGS = False -except ImportError: - terminal = None - HAS_BLESSINGS = False - -REPO_ROOT = pathlib.Path(__file__).parent.parent - - -def exit_invoke(exitcode: int, message: str | None = None, *args: str, **kwargs: str) -> None: - """ - Exit invoke with the passed ``exitcode``. Optionally write a message. - """ - if message is not None: - if exitcode > 0: - warn(message, *args, **kwargs) - else: - info(message, *args, **kwargs) - sys.exit(exitcode) - - -def info(message: str, *args: str, **kwargs: str) -> None: - """ - Write an information message. - """ - if not isinstance(message, str): - message = str(message) - message = message.format(*args, **kwargs) - if terminal: - message = terminal.bold(terminal.green(message)) - write_message(message) - - -def warn(message: str, *args: str, **kwargs: str) -> None: - """ - Write a warning message. - """ - if not isinstance(message, str): - message = str(message) - message = message.format(*args, **kwargs) - if terminal: - message = terminal.bold(terminal.yellow(message)) - write_message(message) - - -def error(message: str, *args: str, **kwargs: str) -> None: - """ - Write an error message. - """ - if not isinstance(message, str): - message = str(message) - message = message.format(*args, **kwargs) - if terminal: - message = terminal.bold(terminal.red(message)) - write_message(message) - - -def write_message(message: str) -> None: - """ - Write a message to ``sys.stderr``. - """ - sys.stderr.write(message) - if not message.endswith("\n"): - sys.stderr.write("\n") - sys.stderr.flush() diff --git a/tools/__init__.py b/tools/__init__.py index daa4996..8d5b574 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -1,9 +1,20 @@ from __future__ import annotations import logging +import pathlib import ptscripts +from ptscripts.parser import DefaultRequirementsConfig + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +DEFAULT_REQS_CONFIG = DefaultRequirementsConfig( + requirements_files=[REPO_ROOT / "requirements.txt"], +) + +ptscripts.set_default_requirements_config(DEFAULT_REQS_CONFIG) + +ptscripts.register_tools_module("tools.ci") ptscripts.register_tools_module("tools.pre_commit") for name in ("boto3", "botocore", "urllib3"): diff --git a/tools/ci.py b/tools/ci.py new file mode 100644 index 0000000..040f33c --- /dev/null +++ b/tools/ci.py @@ -0,0 +1,144 @@ +""" +These commands are used by pre-commit. +""" +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import json +import logging +import os +import pathlib +import shutil + +from ptscripts import command_group +from ptscripts import Context + +import tools.utils + +log = logging.getLogger(__name__) + +# Define the command group +cgroup = command_group(name="ci", help="CI Related Commands", description=__doc__) + + +@cgroup.command( + name="create-manifest-list-and-push", + arguments={ + "container": { + "help": "Container name", + }, + "digests": { + "help": "Digests path", + }, + }, +) +def create_manifest_list_and_push(ctx: Context, container: str, digests: pathlib.Path): + """ + Create manifest list and push. + """ + docker = shutil.which("docker") + if not docker: + ctx.warn("Could not find the 'docker' binary") + ctx.exit(0) + cmdline = [docker, "buildx", "imagetools" "create"] + if "DOCKER_METADATA_OUTPUT_JSON" not in os.environ: + ctx.error("The 'DOCKER_METADATA_OUTPUT_JSON' environment variable is not set") + ctx.exit(1) + + try: + data = json.loads(os.environ["DOCKER_METADATA_OUTPUT_JSON"]) + except ValueError: + ctx.error("Failed to load JSON from the 'DOCKER_METADATA_OUTPUT_JSON' environment variable") + ctx.exit(1) + + for tag in data["tags"]: + cmdline.append(tag) + + for fpath in digests.iterdir(): + cmdline.append(f"{container}@sha256:{fpath.name}") + + ctx.run(*cmdline) + + +@cgroup.command( + name="matrix", + arguments={ + "image": { + "help": "Image name", + }, + "build_platforms": { + "help": "The platforms to build. Defaults to 'linux/amd64' and 'linux/arm64/v8' if not passed", + }, + }, +) +def matrix(ctx: Context, image: str, build_platforms: list[str] = None): + """ + Generate the container mirrors. + """ + mirrors_path = tools.utils.REPO_ROOT / image + + docker = shutil.which("docker") + if not docker: + ctx.warn("Could not find the 'docker' binary") + ctx.exit(0) + + if build_platforms is None: + build_platforms = ["linux/amd64", "linux/arm64/v8"] + + for name, details in tools.utils.get_containers(): + if details["path"] == image: + break + else: + ctx.error(f"Failed to find a container matching path {image}") + ctx.exit(1) + output = [] + tags = [] + for fpath in mirrors_path.glob("*.Dockerfile"): + tags.append(fpath.stem) + if "container" in details: + source_tag = details.get("source_tag", "{version}").format(version=fpath.stem) + source_container = f"{details['container']}:{source_tag}" + else: + source_container = tools.utils.get_source_container(ctx, fpath) + + ret = ctx.run( + docker, "buildx", "imagetools", "inspect", "--raw", source_container, capture=True + ) + data = json.loads(ret.stdout.strip().decode()) + for entry in data.get("manifests", ()): + platform = "{os}/{architecture}".format(**entry["platform"]) + if "variant" in entry["platform"]: + platform += f"/{entry['platform']['variant']}" + if platform not in build_platforms: + continue + output.append( + { + "name": details["name"], + "tag": fpath.stem, + "platform": platform, + "file": str(fpath.relative_to(tools.utils.REPO_ROOT)), + "source_container": source_container, + } + ) + + if not output: + # This is because the buildx inspect did not return anything + output.append( + { + "name": details["name"], + "tag": fpath.stem, + "file": str(fpath.relative_to(tools.utils.REPO_ROOT)), + "source_container": source_container, + "platform": "", + } + ) + + ctx.info("Generated Matrix:") + ctx.print(output) + + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output is not None: + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"dockerinfo={json.dumps(output)}\n") + wfh.write(f"tags={json.dumps(tags)}\n") + wfh.write(f"name={details['name']}") diff --git a/tools/pre_commit.py b/tools/pre_commit.py index 307b586..c8bde70 100644 --- a/tools/pre_commit.py +++ b/tools/pre_commit.py @@ -7,9 +7,12 @@ import logging import shutil +import jinja2.sandbox from ptscripts import command_group from ptscripts import Context +import tools.utils + log = logging.getLogger(__name__) # Define the command group @@ -47,3 +50,167 @@ def actionlint(ctx: Context, files: list[str], no_color: bool = False): cmdline.append(f"-pyflakes={pyflakes}") ret = ctx.run(*cmdline, *files, check=False) ctx.exit(ret.returncode) + + +@cgroup.command( + name="generate-workflows", + arguments={ + "repository": { + "help": "The docker repository to use", + }, + }, +) +def generate_workflows(ctx: Context, repository: str = "saltstack/salt-ci-containers"): + """ + Generate the container mirrors. + """ + containers = tools.utils.get_containers() + + main_readme = tools.utils.REPO_ROOT / "README.md" + main_readme_contents = [] + + for line in main_readme.read_text().splitlines(): + if line == "": + main_readme_contents.append(line) + main_readme_contents.append("\n## Salt Releases") + break + else: + main_readme_contents.append(line) + + ctx.run( + "git", + "rm", + "-f", + "mirrors/*/*.Dockerfile", + ".github/workflows/*-containers.yml", + check=False, + ) + for path in tools.utils.REPO_ROOT.joinpath("mirrors").glob("*"): + if list(path.glob("*")) == [path / "README.md"]: + ctx.run("git", "rm", "-rf", str(path), check=False) + + custom_headers_included = False + mirrors_header_included = False + cron_hour_range = list(range(0, 24)) + for name, details in containers: + is_mirror = details["is_mirror"] + + if is_mirror: + if not mirrors_header_included: + main_readme_contents.append("\n\n## Mirrors") + mirrors_header_included = True + + ctx.info(f"Generating {name} mirror...") + container = details["container"] + if "/" in container: + org, container_name = container.rsplit("/", 1) + else: + org = "_" + container_name = container + + source_tag = details.get("source_tag") + container_dir = ( + tools.utils.REPO_ROOT / "mirrors" / (details.get("dest") or details["name"]) + ) + container_dir.mkdir(parents=True, exist_ok=True) + else: + org = repository + container_name = details["name"] + if details["name"] == "salt": + container_dir = tools.utils.REPO_ROOT / container_name + else: + if not custom_headers_included: + main_readme_contents.append("\n## Custom") + custom_headers_included = True + container_dir = tools.utils.REPO_ROOT / "custom" / container_name + + readme = container_dir / "README.md" + readme_contents = [] + for version in sorted(details["versions"]): + ctx.info(f" Generating docker file for version {version}...") + dockerfile = container_dir / f"{version}.Dockerfile" + if is_mirror: + local_source_tag = source_tag + if local_source_tag is not None: + local_source_tag = local_source_tag.format(version=version) + dockerfile_exists = dockerfile.exists() + if container.count("/") > 1: + hosting = f"https://{org}/{container_name}" + if "quay" in org: + hosting += f"?tab=tags&tag={local_source_tag or version}" + else: + hosting = f"https://hub.docker.com/r/{org}/{container_name}/tags?name={local_source_tag or version}" + readme_contents.append( + f"- [{org if org != '_' else 'dockerhub'}/{container_name}:{local_source_tag or version}]" + f"({hosting}) - `ghcr.io/{repository}/{details['name']}:{version}`" + ) + source_container = f"{container}:{local_source_tag or version}" + with dockerfile.open("w") as wfh: + wfh.write(f"FROM {source_container}\n") + for command in details.get("commands", ()): + wfh.write(f"RUN {command}\n") + if not dockerfile_exists: + ctx.run("git", "add", str(dockerfile.relative_to(tools.utils.REPO_ROOT))) + else: + source_container = tools.utils.get_source_container(ctx, dockerfile) + readme_contents.append( + f"- {container_name}:{version} - `ghcr.io/{repository}/{container_name}:{version}`" + ) + + readme_exists = readme.exists() + workflow_file_name = f"{details.get('dest') or details['name']}-containers.yml" + with readme.open("w") as wfh: + header = ( + f"# [![{name}]" + f"(https://github.com/{repository}/actions/workflows/{workflow_file_name}/badge.svg)]" + f"(https://github.com/{repository}/actions/workflows/{workflow_file_name})\n" + ) + main_readme_contents.append("\n") + main_readme_contents.append(f"##{header}") + main_readme_contents.extend(readme_contents) + wfh.write(f"{header}\n") + wfh.write("\n".join(readme_contents)) + wfh.write("\n") + + if not readme_exists: + ctx.run("git", "add", str(readme.relative_to(tools.utils.REPO_ROOT))) + + ctx.info(f" Generating Github workflow for {name} mirror...") + env = jinja2.sandbox.SandboxedEnvironment( + block_start_string="<%", + block_end_string="%>", + variable_start_string="<{", + variable_end_string="}>", + extensions=[ + "jinja2.ext.do", + ], + ) + workflow_tpl = tools.utils.REPO_ROOT / ".github" / "workflows" / ".container.template.j2" + template = env.from_string(workflow_tpl.read_text()) + cron_hour = cron_hour_range.pop() + if not cron_hour_range: + cron_hour_range = list(range(0, 24)) + jinja_context = { + "name": name, + "slug": details["name"], + "repository_path": container_dir.relative_to(tools.utils.REPO_ROOT), + "is_mirror": is_mirror, + "workflow_file_name": workflow_file_name, + "cron_hour": cron_hour, + } + workflows_dir = tools.utils.REPO_ROOT / ".github" / "workflows" + workflow_path = workflows_dir / workflow_file_name + workflow_path_exists = workflow_path.exists() + ctx.info(f" Writing {workflow_path.relative_to(tools.utils.REPO_ROOT)} ...") + workflow_path.write_text(template.render(**jinja_context).rstrip() + "\n") + if not workflow_path_exists: + ctx.run("git", "add", str(workflow_path.relative_to(tools.utils.REPO_ROOT))) + + main_readme_contents[-1] = main_readme_contents[-1].rstrip() + main_readme_contents.append("\n") + + with main_readme.open("w") as wfh: + contents = "\n".join(main_readme_contents).rstrip() + wfh.write(f"{contents}\n") + + ctx.run("git", "add", ".github/workflows/*-containers.yml") diff --git a/tools/utils.py b/tools/utils.py new file mode 100644 index 0000000..0b53c78 --- /dev/null +++ b/tools/utils.py @@ -0,0 +1,67 @@ +""" +Tools utilities. +""" +from __future__ import annotations + +import os +import pathlib + +import yaml + +REPO_ROOT = pathlib.Path(__file__).parent.parent + + +def get_containers(): + """ + Return a list of containers. + """ + containers_path = REPO_ROOT / "containers.yml" + if containers_path.exists(): + with containers_path.open("r") as rfh: + loaded_containers = yaml.safe_load(rfh.read()) + else: + loaded_containers = { + "salt": {}, + "custom": {}, + "mirrors": {}, + } + + salt_containers = loaded_containers["salt"] + custom_containers = loaded_containers["custom"] + mirror_containers = loaded_containers["mirrors"] + for name, details in salt_containers.items(): + details["is_mirror"] = False + details["path"] = "salt" + salt_containers[name] = details + + for name, details in custom_containers.items(): + details["is_mirror"] = False + details["path"] = os.path.join("custom", details.get("dest") or details["name"]) + custom_containers[name] = details + + for name, details in mirror_containers.items(): + details["is_mirror"] = True + if "name" not in details: + details["name"] = name.lower().replace(" ", "-") + details["path"] = os.path.join("mirrors", details.get("dest") or details["name"]) + mirror_containers[name] = details + + return list(sorted(salt_containers.items()) + sorted(custom_containers.items())) + list( + sorted(mirror_containers.items()) + ) + + +def get_source_container(ctx, dockerfile): + """ + Returns the source container. + """ + source_container = None + with dockerfile.open("r") as rfh: + for line in rfh: + if line.startswith("FROM "): + source_container = line.strip().split()[-1] + break + else: + ctx.error("Failed to find 'FROM ' line") + ctx.exit(1) + return source_container