Skip to content

CI (pr: #67110)

CI (pr: #67110) #15144

Workflow file for this run

# Do not edit these workflows directly as the changes made will be overwritten.
# Instead, edit the template '.github/workflows/templates/ci.yml.jinja'
---
name: CI
run-name: "CI (${{ github.event_name == 'pull_request' && format('pr: #{0}', github.event.number) || format('{0}: {1}', startsWith(github.event.ref, 'refs/tags') && 'tag' || 'branch', github.ref_name) }})"
on:
push: {}
pull_request:
types:
- labeled
- unlabeled
- opened
- reopened
- synchronize
env:
COLUMNS: 190
CACHE_SEED: SEED-1 # Bump the number to invalidate all caches
RELENV_DATA: "${{ github.workspace }}/.relenv"
PIP_DISABLE_PIP_VERSION_CHECK: "1"
RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
permissions:
contents: read # for dorny/paths-filter to fetch a list of changed files
pull-requests: read # for dorny/paths-filter to read pull requests
actions: read # for technote-space/workflow-conclusion-action to get the job statuses
concurrency:
# Concurrency is defined in a way that concurrent builds against branches do
# not cancel previous builds.
# However, for every new build against the same pull request source branch,
# all older builds against that same branch get canceled.
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.repository }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
prepare-workflow:
name: Prepare Workflow Run
runs-on: ubuntu-22.04
environment: ci
outputs:
jobs: ${{ steps.define-jobs.outputs.jobs }}
changed-files: ${{ steps.process-changed-files.outputs.changed-files }}
testrun: ${{ steps.define-testrun.outputs.testrun }}
salt-version: ${{ steps.setup-salt-version.outputs.salt-version }}
cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }}
latest-release: ${{ steps.get-salt-releases.outputs.latest-release }}
releases: ${{ steps.get-salt-releases.outputs.releases }}
testing-releases: ${{ steps.get-testing-releases.outputs.testing-releases }}
nox-archive-hash: ${{ steps.nox-archive-hash.outputs.nox-archive-hash }}
config: ${{ steps.workflow-config.outputs.config }}
env:
LINUX_ARM_RUNNER: ${{ vars.LINUX_ARM_RUNNER }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # Full clone to also get the tags to get the right salt version
- name: Get Changed Files
if: ${{ github.event_name == 'pull_request'}}
id: changed-files
uses: dorny/paths-filter@v3
with:
token: ${{ github.token }}
list-files: json
filters: |
repo:
- added|modified:
- '**'
doc-requirements:
- added|modified: &doc_requirements
- requirements/static/ci/py3.*/docs.txt
lint-requirements:
- added|modified: &lint_requirements
- requirements/static/ci/py3.*/lint.txt
pkg_requirements:
- added|modified: &pkg_requirements
- requirements/static/pkg/py3.*/darwin.txt
- requirements/static/pkg/py3.*/linux.txt
- requirements/static/pkg/py3.*/freebsd.txt
- requirements/static/pkg/py3.*/windows.txt
test_requirements:
- added|modified: &test_requirements
- requirements/static/ci/py3.*/darwin.txt
- requirements/static/ci/py3.*/linux.txt
- requirements/static/ci/py3.*/freebsd.txt
- requirements/static/ci/py3.*/windows.txt
- requirements/static/ci/py3.*/darwin-crypto.txt
- requirements/static/ci/py3.*/linux-crypto.txt
- requirements/static/ci/py3.*/freebsd-crypto.txt
- requirements/static/ci/py3.*/windows-crypto.txt
deleted:
- deleted:
- '**'
docs:
- added|modified:
- doc/**
- .github/workflows/build-docs.yml
- *doc_requirements
workflows:
- added|modified:
- cicd/shared-gh-workflows-context.yml
- .github/actions/**/action.yml
- .github/workflows/*.yml
- .github/workflows/templates/*.yml.jinja2
- tools/precommit/workflows.py
salt:
- added|modified: &salt_added_modified
- setup.py
- noxfile.py
- salt/**/*.py
- tasks/**/*.py
- tools/**/*.py
tests:
- added|modified: &tests_added_modified
- tests/**/*.py
lint:
- added|modified:
- .pylintrc
- *lint_requirements
golden_images:
- added|modified:
- cicd/golden-images.json
pkg_tests:
- added|modified: &pkg_tests_added_modified
- pkg/**
- *pkg_requirements
- *salt_added_modified
nsis_tests:
- added|modified: &nsis_tests
- pkg/windows/nsis/**
testrun:
- added|modified:
- *pkg_requirements
- *test_requirements
- *salt_added_modified
- *tests_added_modified
- *pkg_tests_added_modified
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Setup Python Tools Scripts
uses: ./.github/actions/setup-python-tools-scripts
with:
cache-prefix: ${{ env.CACHE_SEED }}
- name: Pretty Print The GH Actions Event
run:
tools ci print-gh-event
- name: Set Cache Seed Output
id: set-cache-seed
run: |
tools ci define-cache-seed ${{ env.CACHE_SEED }}
- name: Setup Salt Version
id: setup-salt-version
uses: ./.github/actions/setup-salt-version
with:
salt-version: ""
validate-version: true
- name: Get Hash For Nox Tarball Cache
id: nox-archive-hash
run: |
echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py', 'pkg/common/env-cleanup-rules.yml', '.github/workflows/build-deps-ci-action.yml') }}" | tee -a "$GITHUB_OUTPUT"
- name: Write Changed Files To A Local File
run:
echo '${{ toJSON(steps.changed-files.outputs) }}' > changed-files.json
- name: Check Local Changed Files Contents
if: ${{ github.event_name == 'pull_request' }}
run:
cat changed-files.json
- name: Process Changed Files
id: process-changed-files
run: |
tools ci process-changed-files ${{ github.event_name }} changed-files.json
- name: Check Collected Changed Files
if: ${{ github.event_name == 'pull_request' }}
run: |
echo '${{ steps.process-changed-files.outputs.changed-files }}' | jq -C '.'
- name: Define Jobs To Run
id: define-jobs
run: |
tools ci define-jobs ${{ github.event_name }} changed-files.json
- name: Get Salt Releases
id: get-salt-releases
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
tools ci get-releases
- name: Get Latest Salt Releases for Testing
id: get-testing-releases
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
tools ci get-testing-releases ${{ join(fromJSON(steps.get-salt-releases.outputs.releases), ' ') }} --salt-version ${{ steps.setup-salt-version.outputs.salt-version }}
- name: Define Testrun
id: define-testrun
run: |
tools ci define-testrun ${{ github.event_name }} changed-files.json
- name: Define workflow config
id: workflow-config
run: |
tools ci workflow-config ${{ steps.setup-salt-version.outputs.salt-version }} ${{ github.event_name }} changed-files.json
- name: Check Contents of generated testrun-changed-files.txt
if: ${{ fromJSON(steps.define-testrun.outputs.testrun)['type'] != 'full' }}
run: |
cat testrun-changed-files.txt || true
- name: Upload testrun-changed-files.txt
if: ${{ fromJSON(steps.define-testrun.outputs.testrun)['type'] != 'full' }}
uses: actions/upload-artifact@v4
with:
name: testrun-changed-files.txt
path: testrun-changed-files.txt
pre-commit:
name: Pre-Commit
uses: ./.github/workflows/pre-commit-action.yml
needs:
- prepare-workflow
with:
cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
changed-files: ${{ needs.prepare-workflow.outputs.changed-files }}
pre-commit-version: "3.0.4"
lint:
name: Lint
if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['lint'] }}
uses: ./.github/workflows/lint-action.yml
needs:
- prepare-workflow
with:
changed-files: ${{ needs.prepare-workflow.outputs.changed-files }}
nsis-tests:
name: NSIS Tests
uses: ./.github/workflows/nsis-tests.yml
needs:
- prepare-workflow
with:
changed-files: ${{ needs.prepare-workflow.outputs.changed-files }}
prepare-release:
name: "Prepare Release: ${{ needs.prepare-workflow.outputs.salt-version }}"
runs-on:
- ubuntu-22.04
if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['prepare-release'] }}
needs:
- prepare-workflow
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Setup Python Tools Scripts
uses: ./.github/actions/setup-python-tools-scripts
with:
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}-changelog
- name: Setup Salt Version
id: setup-salt-version
uses: ./.github/actions/setup-salt-version
with:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
- name: Update Debian changelog
shell: bash
if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
run: |
tools changelog update-deb --draft
tools changelog update-deb
- name: Update RPM changelog
shell: bash
if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
run: |
tools changelog update-rpm --draft
tools changelog update-rpm
- name: Update Release Notes
shell: bash
if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
run: |
tools changelog update-release-notes --draft
tools changelog update-release-notes
- name: Generate MAN Pages
shell: bash
if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
env:
LATEST_RELEASE: "${{ needs.prepare-workflow.outputs.salt-version }}"
SALT_ON_SALTSTACK: "1"
run: |
tools docs man
- name: Update Changelog
shell: bash
if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
run: |
tools changelog update-changelog-md --draft
tools changelog update-changelog-md
- name: Show Changes Diff
shell: bash
if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
run: |
git diff --color
- name: Configure Git
shell: bash
if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
run: |
git config --global user.name "Salt Project Packaging"
git config --global user.email saltproject-packaging@vmware.com
- name: Setup Pre-Commit
if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
uses: ./.github/actions/setup-pre-commit
with:
version: "3.0.4"
cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
- name: Commit Changes
shell: bash
if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
env:
SKIP: lint-salt,lint-tests
PRE_COMMIT_COLOR: always
run: |
# Run it twice so that pre-commit can fix anything that can be automatically fixed.
git commit -am "Release v${{ needs.prepare-workflow.outputs.salt-version }}" || \
git commit -am "Release v${{ needs.prepare-workflow.outputs.salt-version }}"
- name: Create release changes patch
shell: bash
if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
run: |
git format-patch --keep-subject --binary --stdout HEAD^ > salt-${{ needs.prepare-workflow.outputs.salt-version }}.patch
- name: Upload Changes Diff Artifact
uses: actions/upload-artifact@v4
if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
with:
name: salt-${{ needs.prepare-workflow.outputs.salt-version }}.patch
path: salt-${{ needs.prepare-workflow.outputs.salt-version }}.patch
retention-days: 7
if-no-files-found: error
build-docs:
name: Documentation
if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-docs'] }}
needs:
- prepare-workflow
- build-source-tarball
uses: ./.github/workflows/build-docs.yml
with:
cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
build-source-tarball:
name: Build Source Tarball
if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-source-tarball'] }}
needs:
- prepare-workflow
- prepare-release
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Setup Python Tools Scripts
uses: ./.github/actions/setup-python-tools-scripts
with:
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}-build
- name: Setup Salt Version
id: setup-salt-version
uses: ./.github/actions/setup-salt-version
with:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
- name: Build Source Tarball
uses: ./.github/actions/build-source-tarball
with:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
build-deps-onedir:
name: Build Onedir Dependencies
if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir'] }}
needs:
- prepare-workflow
uses: ./.github/workflows/build-deps-onedir.yml
with:
cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
relenv-version: "0.18.0"
python-version: "3.10.15"
matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
build-salt-onedir:
name: Build Salt Onedir
if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir'] }}
needs:
- prepare-workflow
- build-deps-onedir
- build-source-tarball
uses: ./.github/workflows/build-salt-onedir.yml
with:
cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
relenv-version: "0.18.0"
python-version: "3.10.15"
matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
build-pkgs-onedir:
name: Build Packages
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] }}
needs:
- prepare-workflow
- build-salt-onedir
uses: ./.github/workflows/build-packages.yml
with:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}
relenv-version: "0.18.0"
python-version: "3.10.15"
source: "onedir"
matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
build-ci-deps:
name: CI Deps
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-ci'] }}
needs:
- prepare-workflow
- build-salt-onedir
uses: ./.github/workflows/build-deps-ci-action.yml
with:
nox-session: ci-test-onedir
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
nox-archive-hash: "${{ needs.prepare-workflow.outputs.nox-archive-hash }}"
matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
test-packages:
name: Test Package
if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['test-pkg'] }}
needs:
- prepare-workflow
- build-pkgs-onedir
- build-ci-deps
uses: ./.github/workflows/test-packages-action.yml
with:
nox-session: ci-test-onedir
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
nox-version: 2022.8.7
python-version: "3.10"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }}
testing-releases: ${{ needs.prepare-workflow.outputs.testing-releases }}
matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['pkg-test-matrix']) }}
test:
name: Test Salt
if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['test'] }}
needs:
- prepare-workflow
- build-ci-deps
uses: ./.github/workflows/test-action.yml
with:
nox-session: ci-test-onedir
nox-version: 2022.8.7
python-version: "3.10"
testrun: ${{ needs.prepare-workflow.outputs.testrun }}
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }}
workflow-slug: ci
default-timeout: 180
matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['test-matrix']) }}
combine-all-code-coverage:
name: Combine Code Coverage
if: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] == false }}
runs-on: ubuntu-22.04
env:
PIP_INDEX_URL: https://pypi.org/simple
needs:
- prepare-workflow
- build-ci-deps
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Setup Python Tools Scripts
id: python-tools-scripts
uses: ./.github/actions/setup-python-tools-scripts
with:
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}-coverage
- name: Install Nox
run: |
python3 -m pip install 'nox==2022.8.7'
- name: Merge All Code Coverage Test Run Artifacts
continue-on-error: true
uses: actions/upload-artifact/merge@v4
with:
name: all-testrun-coverage-artifacts
pattern: all-testrun-coverage-artifacts-*
separate-directories: false
delete-merged: true
- name: Get coverage reports
id: get-coverage-reports
uses: actions/download-artifact@v4
with:
path: artifacts/coverage/
pattern: all-testrun-coverage-artifacts*
merge-multiple: true
- name: Display structure of downloaded files
run: tree -a artifacts/
- name: Install Codecov CLI
run: |
# We can't yet use tokenless uploads with the codecov CLI
# python3 -m pip install codecov-cli
#
curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --import
curl -Os https://uploader.codecov.io/latest/linux/codecov
curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM
curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig
gpg --verify codecov.SHA256SUM.sig codecov.SHA256SUM
shasum -a 256 -c codecov.SHA256SUM
chmod +x codecov
mv ./codecov /usr/local/bin/
- name: Create XML Coverage Reports
run: |
nox --force-color -e create-xml-coverage-reports
- name: Upload Code Coverage To Codecov
if: ${{ ! github.event.repository.private && ! github.event.repository.fork }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
run: |
tools ci upload-coverage --commit-sha=${{ github.event.pull_request.head.sha || github.sha }} artifacts/coverage/
- name: Combine Code Coverage
run: |
nox --force-color -e combine-coverage
- name: Report Salt Code Coverage
run: |
nox --force-color -e coverage-report -- salt
- name: Create Salt Code Coverage HTML Report
run: |
nox --force-color -e create-html-coverage-report -- salt
- name: Create Salt Code Coverage HTML Report
run: |
nox --force-color -e create-html-coverage-report -- salt
- name: Upload Salt Code Coverage HTML Report
uses: actions/upload-artifact@v4
with:
name: code-coverage-salt-html-report
path: artifacts/coverage/html/salt
retention-days: 7
if-no-files-found: error
include-hidden-files: true
- name: Report Combined Code Coverage
run: |
nox --force-color -e coverage-report
- name: Create Combined Code Coverage JSON Report
run: |
nox --force-color -e create-json-coverage-reports
- name: Upload Combined Code Coverage JSON Report
uses: actions/upload-artifact@v4
with:
name: code-coverage-full-json-report
path: artifacts/coverage/coverage.json
retention-days: 7
if-no-files-found: error
include-hidden-files: true
- name: Create Combined Code Coverage HTML Report
run: |
nox --force-color -e create-html-coverage-report
- name: Upload Combined Code Coverage HTML Report
uses: actions/upload-artifact@v4
with:
name: code-coverage-full-html-report
path: artifacts/coverage/html/full
retention-days: 7
if-no-files-found: error
include-hidden-files: true
set-pipeline-exit-status:
# This step is just so we can make github require this step, to pass checks
# on a pull request instead of requiring all
name: Set the ${{ github.workflow }} Pipeline Exit Status
if: always()
runs-on: ubuntu-22.04
needs:
- prepare-workflow
- pre-commit
- lint
- nsis-tests
- build-docs
- build-deps-onedir
- build-salt-onedir
- combine-all-code-coverage
- build-ci-deps
steps:
- name: Get workflow information
id: get-workflow-info
uses: im-open/workflow-conclusion@v2
- name: Set Pipeline Exit Status
shell: bash
run: |
if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then
exit 1
else
exit 0
fi
- name: Done
if: always()
run:
echo "All worflows finished"