ci.yml

# Do not edit these workflows directly as the changes made will be overwritten.
# Instead, edit the template '.github/workflows/templates/ci.yml.jinja'
---
name: CI
run-name: "CI (${{ github.event_name == 'pull_request' && format('pr: #{0}', github.event.number) || format('{0}: {1}', startsWith(github.event.ref, 'refs/tags') && 'tag' || 'branch', github.ref_name) }})"

on:
  push: {}
  pull_request:
    types:
      - labeled
      - unlabeled
      - opened
      - reopened
      - synchronize

env:
  COLUMNS: 190
  CACHE_SEED: SEED-1  # Bump the number to invalidate all caches
  RELENV_DATA: "${{ github.workspace }}/.relenv"
  PIP_DISABLE_PIP_VERSION_CHECK: "1"
  RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"

permissions:
  contents: read  # for dorny/paths-filter to fetch a list of changed files
  pull-requests: read  # for dorny/paths-filter to read pull requests
  actions: read  # for technote-space/workflow-conclusion-action to get the job statuses

concurrency:
  # Concurrency is defined in a way that concurrent builds against branches do
  # not cancel previous builds.
  # However, for every new build against the same pull request source branch,
  # all older builds against that same branch get canceled.
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.repository }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:

  prepare-workflow:
    name: Prepare Workflow Run
    runs-on: ubuntu-22.04
    outputs:
      jobs: ${{ steps.define-jobs.outputs.jobs }}
      changed-files: ${{ steps.process-changed-files.outputs.changed-files }}
      testrun: ${{ steps.define-testrun.outputs.testrun }}
      salt-version: ${{ steps.setup-salt-version.outputs.salt-version }}
      cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }}
      latest-release: ${{ steps.get-salt-releases.outputs.latest-release }}
      releases: ${{ steps.get-salt-releases.outputs.releases }}
      testing-releases: ${{ steps.get-testing-releases.outputs.testing-releases }}
      nox-archive-hash: ${{ steps.nox-archive-hash.outputs.nox-archive-hash }}
      config: ${{ steps.workflow-config.outputs.config }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0  # Full clone to also get the tags to get the right salt version

      - name: Get Changed Files
        if: ${{ github.event_name == 'pull_request'}}
        id: changed-files
        uses: dorny/paths-filter@v3
        with:
          token: ${{ github.token }}
          list-files: json
          filters: |
            repo:
              - added|modified:
                - '**'
            doc-requirements:
              - added|modified: &doc_requirements
                - requirements/static/ci/py3.*/docs.txt
            lint-requirements:
              - added|modified: &lint_requirements
                - requirements/static/ci/py3.*/lint.txt
            pkg_requirements:
              - added|modified: &pkg_requirements
                - requirements/static/pkg/py3.*/darwin.txt
                - requirements/static/pkg/py3.*/linux.txt
                - requirements/static/pkg/py3.*/freebsd.txt
                - requirements/static/pkg/py3.*/windows.txt
            test_requirements:
              - added|modified: &test_requirements
                - requirements/static/ci/py3.*/darwin.txt
                - requirements/static/ci/py3.*/linux.txt
                - requirements/static/ci/py3.*/freebsd.txt
                - requirements/static/ci/py3.*/windows.txt
                - requirements/static/ci/py3.*/darwin-crypto.txt
                - requirements/static/ci/py3.*/linux-crypto.txt
                - requirements/static/ci/py3.*/freebsd-crypto.txt
                - requirements/static/ci/py3.*/windows-crypto.txt
            deleted:
              - deleted:
                - '**'
            docs:
              - added|modified:
                - doc/**
                - .github/workflows/build-docs.yml
                - *doc_requirements
            workflows:
              - added|modified:
                - cicd/shared-gh-workflows-context.yml
                - .github/actions/**/action.yml
                - .github/workflows/*.yml
                - .github/workflows/templates/*.yml.jinja2
                - tools/precommit/workflows.py
            salt:
              - added|modified: &salt_added_modified
                - setup.py
                - noxfile.py
                - salt/**/*.py
                - tasks/**/*.py
                - tools/**/*.py
            tests:
              - added|modified: &tests_added_modified
                - tests/**/*.py
            lint:
              - added|modified:
                - .pylintrc
                - *lint_requirements
            golden_images:
              - added|modified:
                - cicd/golden-images.json
            pkg_tests:
              - added|modified: &pkg_tests_added_modified
                - pkg/**
                - *pkg_requirements
                - *salt_added_modified
            nsis_tests:
              - added|modified: &nsis_tests
                - pkg/windows/nsis/**
            testrun:
              - added|modified:
                - *pkg_requirements
                - *test_requirements
                - *salt_added_modified
                - *tests_added_modified
                - *pkg_tests_added_modified

      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ env.CACHE_SEED }}

      - name: Pretty Print The GH Actions Event
        run:
          tools ci print-gh-event

      - name: Set Cache Seed Output
        id: set-cache-seed
        run: |
          tools ci define-cache-seed ${{ env.CACHE_SEED }}

      - name: Setup Salt Version
        id: setup-salt-version
        uses: ./.github/actions/setup-salt-version
        with:
          salt-version: ""
          validate-version: true

      - name: Get Hash For Nox Tarball Cache
        id: nox-archive-hash
        run: |
          echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py', 'pkg/common/env-cleanup-rules.yml', '.github/workflows/build-deps-ci-action.yml') }}" | tee -a "$GITHUB_OUTPUT"

      - name: Write Changed Files To A Local File
        run:
          echo '${{ toJSON(steps.changed-files.outputs) }}' > changed-files.json

      - name: Check Local Changed Files Contents
        if: ${{ github.event_name == 'pull_request' }}
        run:
          cat changed-files.json

      - name: Process Changed Files
        id: process-changed-files
        run: |
          tools ci process-changed-files ${{ github.event_name }} changed-files.json

      - name: Check Collected Changed Files
        if: ${{ github.event_name == 'pull_request' }}
        run: |
          echo '${{ steps.process-changed-files.outputs.changed-files }}' | jq -C '.'

      - name: Define Jobs To Run
        id: define-jobs
        run: |
          tools ci define-jobs ${{ github.event_name }} changed-files.json

      - name: Get Salt Releases
        id: get-salt-releases
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          tools ci get-releases

      - name: Get Latest Salt Releases for Testing
        id: get-testing-releases
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          tools ci get-testing-releases ${{ join(fromJSON(steps.get-salt-releases.outputs.releases), ' ') }} --salt-version ${{ steps.setup-salt-version.outputs.salt-version }}

      - name: Define Testrun
        id: define-testrun
        run: |
          tools ci define-testrun ${{ github.event_name }} changed-files.json

      - name: Define workflow config
        id: workflow-config
        run: |
          tools ci workflow-config ${{ steps.setup-salt-version.outputs.salt-version }} ${{ github.event_name }} changed-files.json

      - name: Check Contents of generated testrun-changed-files.txt
        if: ${{ fromJSON(steps.define-testrun.outputs.testrun)['type'] != 'full' }}
        run: |
          cat testrun-changed-files.txt || true

      - name: Upload testrun-changed-files.txt
        if: ${{ fromJSON(steps.define-testrun.outputs.testrun)['type'] != 'full' }}
        uses: actions/upload-artifact@v4
        with:
          name: testrun-changed-files.txt
          path: testrun-changed-files.txt


  pre-commit:
    name: Pre-Commit
    uses: ./.github/workflows/pre-commit-action.yml
    needs:
      - prepare-workflow
    with:
      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
      changed-files: ${{ needs.prepare-workflow.outputs.changed-files }}
      pre-commit-version: "3.0.4"

  lint:
    name: Lint
    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['lint'] }}
    uses: ./.github/workflows/lint-action.yml
    needs:
      - prepare-workflow
    with:
      changed-files: ${{ needs.prepare-workflow.outputs.changed-files }}
  nsis-tests:
    name: NSIS Tests
    uses: ./.github/workflows/nsis-tests.yml
    needs:
      - prepare-workflow
    with:
      changed-files: ${{ needs.prepare-workflow.outputs.changed-files }}

  prepare-release:
    name: "Prepare Release: ${{ needs.prepare-workflow.outputs.salt-version }}"
    runs-on:
      - ubuntu-22.04
    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['prepare-release'] }}
    needs:
      - prepare-workflow
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}-changelog

      - name: Setup Salt Version
        id: setup-salt-version
        uses: ./.github/actions/setup-salt-version
        with:
          salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"

      - name: Update Debian changelog
        shell: bash
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        run: |
          tools changelog update-deb --draft
          tools changelog update-deb

      - name: Update RPM changelog
        shell: bash
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        run: |
          tools changelog update-rpm --draft
          tools changelog update-rpm

      - name: Update Release Notes
        shell: bash
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        run: |
          tools changelog update-release-notes --draft
          tools changelog update-release-notes

      - name: Generate MAN Pages
        shell: bash
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        env:
          LATEST_RELEASE: "${{ needs.prepare-workflow.outputs.salt-version }}"
          SALT_ON_SALTSTACK: "1"
        run: |
          tools docs man

      - name: Update Changelog
        shell: bash
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        run: |
          tools changelog update-changelog-md --draft
          tools changelog update-changelog-md

      - name: Show Changes Diff
        shell: bash
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        run: |
          git diff --color

      - name: Configure Git
        shell: bash
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        run: |
          git config --global user.name "Salt Project Packaging"
          git config --global user.email saltproject-packaging@vmware.com

      - name: Setup Pre-Commit
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        uses: ./.github/actions/setup-pre-commit
        with:
          version: "3.0.4"
          cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}

      - name: Commit Changes
        shell: bash
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        env:
          SKIP: lint-salt,lint-tests
          PRE_COMMIT_COLOR: always
        run: |
          # Run it twice so that pre-commit can fix anything that can be automatically fixed.
          git commit -am "Release v${{ needs.prepare-workflow.outputs.salt-version }}" || \
            git commit -am "Release v${{ needs.prepare-workflow.outputs.salt-version }}"

      - name: Create release changes patch
        shell: bash
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        run: |
          git format-patch --keep-subject --binary --stdout HEAD^ > salt-${{ needs.prepare-workflow.outputs.salt-version }}.patch

      - name: Upload Changes Diff Artifact
        uses: actions/upload-artifact@v4
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        with:
          name: salt-${{ needs.prepare-workflow.outputs.salt-version }}.patch
          path: salt-${{ needs.prepare-workflow.outputs.salt-version }}.patch
          retention-days: 7
          if-no-files-found: error

  build-docs:
    name: Documentation
    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-docs'] }}
    needs:
      - prepare-workflow
      - build-source-tarball
    uses: ./.github/workflows/build-docs.yml
    with:
      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"

  build-source-tarball:
    name: Build Source Tarball
    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-source-tarball'] }}
    needs:
      - prepare-workflow
      - prepare-release
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}-build

      - name: Setup Salt Version
        id: setup-salt-version
        uses: ./.github/actions/setup-salt-version
        with:
          salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"

      - name: Build Source Tarball
        uses: ./.github/actions/build-source-tarball
        with:
          salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"

  build-deps-onedir:
    name: Build Onedir Dependencies
    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir'] }}
    needs:
      - prepare-workflow
    uses: ./.github/workflows/build-deps-onedir.yml
    with:
      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
      relenv-version: "0.18.0"
      python-version: "3.10.15"
      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}

  build-salt-onedir:
    name: Build Salt Onedir
    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir'] }}
    needs:
      - prepare-workflow
      - build-deps-onedir
      - build-source-tarball
    uses: ./.github/workflows/build-salt-onedir.yml
    with:
      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
      relenv-version: "0.18.0"
      python-version: "3.10.15"
      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}

  build-pkgs-onedir:
    name: Build Packages
    if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] }}
    needs:
      - prepare-workflow
      - build-salt-onedir
    uses: ./.github/workflows/build-packages.yml
    with:
      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}
      relenv-version: "0.18.0"
      python-version: "3.10.15"
      source: "onedir"
      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
  build-ci-deps:
    name: CI Deps
    if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-ci'] }}
    needs:
      - prepare-workflow
      - build-salt-onedir
    uses: ./.github/workflows/build-deps-ci-action.yml
    with:
      nox-session: ci-test-onedir
      nox-version: 2022.8.7
      python-version: "3.10"
      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
      nox-archive-hash: "${{ needs.prepare-workflow.outputs.nox-archive-hash }}"
      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
  test-packages:
    name: Test Package
    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['test-pkg'] }}
    needs:
      - prepare-workflow
      - build-pkgs-onedir
      - build-ci-deps
    uses: ./.github/workflows/test-packages-action.yml
    with:
      nox-session: ci-test-onedir
      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
      nox-version: 2022.8.7
      python-version: "3.10"
      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
      skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }}
      testing-releases: ${{ needs.prepare-workflow.outputs.testing-releases }}
      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['pkg-test-matrix']) }}
  test:
    name: Test Salt
    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['test'] }}
    needs:
      - prepare-workflow
      - build-ci-deps
    uses: ./.github/workflows/test-action.yml
    with:
      nox-session: ci-test-onedir
      nox-version: 2022.8.7
      python-version: "3.10"
      testrun: ${{ needs.prepare-workflow.outputs.testrun }}
      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
      skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }}
      workflow-slug: ci
      default-timeout: 180
      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['test-matrix']) }}

  combine-all-code-coverage:
    name: Combine Code Coverage
    if: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] == false }}
    runs-on: ubuntu-22.04
    env:
      PIP_INDEX_URL: https://pypi.org/simple
    needs:
      - prepare-workflow
      - build-ci-deps
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Setup Python Tools Scripts
        id: python-tools-scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}-coverage

      - name: Install Nox
        run: |
          python3 -m pip install 'nox==2022.8.7'



      - name: Merge All Code Coverage Test Run Artifacts
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: all-testrun-coverage-artifacts
          pattern: all-testrun-coverage-artifacts-*
          separate-directories: false
          delete-merged: true

      - name: Get coverage reports
        id: get-coverage-reports
        uses: actions/download-artifact@v4
        with:
          path: artifacts/coverage/
          pattern: all-testrun-coverage-artifacts*
          merge-multiple: true

      - name: Display structure of downloaded files
        run: tree -a artifacts/

      - name: Install Codecov CLI
        run: |
          # We can't yet use tokenless uploads with the codecov CLI
          # python3 -m pip install codecov-cli
          #
          curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --import
          curl -Os https://uploader.codecov.io/latest/linux/codecov
          curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM
          curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig
          gpg --verify codecov.SHA256SUM.sig codecov.SHA256SUM
          shasum -a 256 -c codecov.SHA256SUM
          chmod +x codecov
          mv ./codecov /usr/local/bin/

      - name: Create XML Coverage Reports
        run: |
          nox --force-color -e create-xml-coverage-reports

      - name: Upload Code Coverage To Codecov
        if: ${{ ! github.event.repository.private && ! github.event.repository.fork }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        run: |
          tools ci upload-coverage --commit-sha=${{ github.event.pull_request.head.sha || github.sha }} artifacts/coverage/

      - name: Combine Code Coverage
        run: |
          nox --force-color -e combine-coverage

      - name: Report Salt Code Coverage
        run: |
          nox --force-color -e coverage-report -- salt

      - name: Create Salt Code Coverage HTML Report
        run: |
          nox --force-color -e create-html-coverage-report -- salt

      - name: Create Salt Code Coverage HTML Report
        run: |
          nox --force-color -e create-html-coverage-report -- salt

      - name: Upload Salt Code Coverage HTML Report
        uses: actions/upload-artifact@v4
        with:
          name: code-coverage-salt-html-report
          path: artifacts/coverage/html/salt
          retention-days: 7
          if-no-files-found: error
          include-hidden-files: true

      - name: Report Combined Code Coverage
        run: |
          nox --force-color -e coverage-report

      - name: Create Combined Code Coverage JSON Report
        run: |
          nox --force-color -e create-json-coverage-reports

      - name: Upload Combined Code Coverage JSON Report
        uses: actions/upload-artifact@v4
        with:
          name: code-coverage-full-json-report
          path: artifacts/coverage/coverage.json
          retention-days: 7
          if-no-files-found: error
          include-hidden-files: true

      - name: Create Combined Code Coverage HTML Report
        run: |
          nox --force-color -e create-html-coverage-report

      - name: Upload Combined Code Coverage HTML Report
        uses: actions/upload-artifact@v4
        with:
          name: code-coverage-full-html-report
          path: artifacts/coverage/html/full
          retention-days: 7
          if-no-files-found: error
          include-hidden-files: true

  set-pipeline-exit-status:
    # This step is just so we can make github require this step, to pass checks
    # on a pull request instead of requiring all
    name: Set the ${{ github.workflow }} Pipeline Exit Status
    if: always()
    runs-on: ubuntu-22.04
    needs:
      - prepare-workflow
      - pre-commit
      - lint
      - nsis-tests
      - build-docs
      - build-deps-onedir
      - build-salt-onedir
      - combine-all-code-coverage
      - build-ci-deps
    steps:
      - name: Get workflow information
        id: get-workflow-info
        uses: im-open/workflow-conclusion@v2

      - name: Set Pipeline Exit Status
        shell: bash
        run: |
          if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then
            exit 1
          else
            exit 0
          fi

      - name: Done
        if: always()
        run:
          echo "All worflows finished"