diff --git a/.github/actions/import-commit-pragmas/action.yml b/.github/actions/import-commit-pragmas/action.yml deleted file mode 100644 index 58d8ecc179d..00000000000 --- a/.github/actions/import-commit-pragmas/action.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: 'Import Commit Pragmas' -description: 'Import Commit Pragmas' -runs: - using: "composite" - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - run: . ci/gha_functions.sh; - git show -s --format=%B | sed -e "s/'/'\"'\"'/g" | - get_commit_pragmas | - sed -e 's/^/CP_/' >> $GITHUB_ENV - shell: bash diff --git a/.github/actions/provision-cluster/action.yml b/.github/actions/provision-cluster/action.yml index a11af0f7339..acee498d0d9 100644 --- a/.github/actions/provision-cluster/action.yml +++ b/.github/actions/provision-cluster/action.yml @@ -19,32 +19,48 @@ runs: STAGE_NAME='${{ env.STAGE_NAME }}' REQID='${{ github.event.pull_request.number }}-${{ github.run_number }}' CP_PRIORITY=${{ env.CP_PRIORITY }} + echo "::group::Provisioning" if ! JENKINS_URL='${{ env.JENKINS_URL }}' \ LABEL='${{ env.LABEL }}' \ PRIORITY=${CP_PRIORITY:-3} \ + COVFN_DISABLED=${COVFN_DISABLED} \ provision_cluster "$STAGE_NAME" "${{ github.run_id }}" \ "${{ runner.name }}" \ "$REQID-${STAGE_NAME// /_}"; then + echo "::endgroup::" echo "Failed to provision cluster" exit 1 fi + echo "::endgroup::" - name: Wait for nodes to be ready if: ${{ inputs.condition }} shell: bash run: . ci/gha_functions.sh; set -eux; - wait_nodes_ready + echo "::group::Waiting for nodes to be ready"; + wait_nodes_ready; + echo "::endgroup::" - name: Post-provision cluster if: ${{ inputs.condition }} shell: bash run: | . ci/gha_functions.sh + set -x inst_repos="${{ env.CP_PR_REPOS }} ${{ github.event.inputs.pr-repos }}" - if [ -z "${{ env.CP_RPM_TEST_VERSION }}" ] && + if [ -n "${{ github.event.inputs.rpm-test-version }}" ]; then + rpm_test_version="${{ github.event.inputs.rpm-test-version }}" + elif [ -n "${{ env.CP_RPM_TEST_VERSION }}" ]; then + rpm_test_version="${{ env.CP_RPM_TEST_VERSION }}" + fi + if [ -z "$rpm_test_version" ] && [[ $inst_repos != *daos@* ]]; then inst_repos+=" daos@PR-${{ github.event.pull_request.number }}" inst_repos+=":${{ github.run_number }}" fi - ARTIFACTS_URL=$ARTIFACTS_URL \ + echo "::group::Post-provision configuration"; + ARTIFACTS_URL="$ARTIFACTS_URL" \ INST_REPOS="$(eval echo "$inst_repos")" \ + CI_RPM_TEST_VERSION="$rpm_test_version" \ + DAOS_VERSION="$DAOS_VERSION" \ DISTRO=$PROVISION_DISTRO ci/provisioning/post_provision_config.sh + echo "::endgroup::" diff --git a/.github/actions/variable-from-pragma/action.yml b/.github/actions/variable-from-pragma/action.yml deleted file mode 100644 index 929ed998657..00000000000 --- a/.github/actions/variable-from-pragma/action.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: 'Set Variable from Commit Pragma' -description: 'Set Variable from Commit Pragma' -inputs: - commit_message: - description: Dequoted commit message - required: true - pragma: - description: Pragma to make a variable from - required: true - default: - description: Default value if pragma is not found - required: false -outputs: - value: - description: The value of the pragma - value: ${{ steps.value.outputs.value }} - -runs: - using: "composite" - steps: - - name: Create Variable - id: value - shell: bash - run: | - . ci/gha_functions.sh - set -eu - eval $(echo '${{ inputs.commit_message }}' | get_commit_pragmas) - echo "value=${${{ inputs.pragma }}:-${{ inputs.default }}}" >> $GITHUB_OUTPUT diff --git a/.github/actions/variables-from-env/action.yml b/.github/actions/variables-from-env/action.yml new file mode 100644 index 00000000000..b6598056993 --- /dev/null +++ b/.github/actions/variables-from-env/action.yml @@ -0,0 +1,57 @@ +name: 'Create Variables from Env Variables' +description: 'Variables from Env Variables' +outputs: + NAME: + value: ${{ steps.NAME.outputs.NAME }} + DISTROS: + value: ${{ steps.DISTROS.outputs.DISTROS }} + EL8_BUILD_VERSION: + value: ${{ steps.EL8_BUILD_VERSION.outputs.EL8_BUILD_VERSION }} + EL9_BUILD_VERSION: + value: ${{ steps.EL9_BUILD_VERSION.outputs.EL9_BUILD_VERSION }} + LEAP15_VERSION: + value: ${{ steps.LEAP15_VERSION.outputs.LEAP15_VERSION }} + PACKAGING_DIR: + value: ${{ steps.PACKAGING_DIR.outputs.PACKAGING_DIR }} + COVFN_DISABLED: + value: ${{ steps.COVFN_DISABLED.outputs.COVFN_DISABLED }} + TEST_TAG: + value: ${{ steps.TEST_TAG.outputs.TEST_TAG }} +runs: + using: "composite" + # What a dumb jobs this is + # Needed because of https://github.com/orgs/community/discussions/26671 + # Ideally want to be able to use: + # with: + # NAME: ${{ env.DISTROS }} + # in the Call-RPM-Build job but the above issue prevents it + steps: + - name: Make outputs from env variables + run: | + echo "Make outputs from env variables" \ + "(https://github.com/orgs/community/discussions/26671)" + shell: bash + - id: NAME + run: echo "NAME=${{ env.NAME }}" >> $GITHUB_OUTPUT + shell: bash + - id: DISTROS + run: echo "DISTROS=${{ env.DISTROS }}" >> $GITHUB_OUTPUT + shell: bash + - id: EL8_BUILD_VERSION + run: echo "EL8_BUILD_VERSION=${{ env.EL8_BUILD_VERSION }}" >> $GITHUB_OUTPUT + shell: bash + - id: EL9_BUILD_VERSION + run: echo "EL9_BUILD_VERSION=${{ env.EL9_BUILD_VERSION }}" >> $GITHUB_OUTPUT + shell: bash + - id: LEAP15_VERSION + run: echo "LEAP15_VERSION=${{ env.LEAP15_VERSION }}" >> $GITHUB_OUTPUT + shell: bash + - id: PACKAGING_DIR + run: echo "PACKAGING_DIR=${{ env.PACKAGING_DIR }}" >> $GITHUB_OUTPUT + shell: bash + - id: COVFN_DISABLED + run: echo "COVFN_DISABLED=${{ env.COVFN_DISABLED }}" >> $GITHUB_OUTPUT + shell: bash + - id: TEST_TAG + run: echo "TEST_TAG=${{ env.TEST_TAG }}" >> $GITHUB_OUTPUT + shell: bash diff --git a/.github/workflows/bullseye-coverage.yml b/.github/workflows/bullseye-coverage.yml new file mode 100644 index 00000000000..4f63b285b37 --- /dev/null +++ b/.github/workflows/bullseye-coverage.yml @@ -0,0 +1,656 @@ +name: Bullseye Code Coverage + +env: + # TODO: we really need to define a list of supported versions (ideally it's no more than 2) + # build is done on the lowest version and test on the highest with a "sanity test" + # stage done on all versions in the list ecept the highest + EL8_BUILD_VERSION: 8.6 + EL8_VERSION: 8.8 + EL9_BUILD_VERSION: 9 + EL9_VERSION: 9 + LEAP15_VERSION: 15.5 + NEXT_VERSION_master: 1000 + # Which distros to build for + DISTROS: el8 + TEST_TAG: pr + PACKAGING_DIR: utils/rpms + NAME: daos + COVFN_DISABLED: false + +on: + workflow_dispatch: + inputs: + pr-repos: + description: 'Any PR-repos that you want included in this build' + required: false + commit-message: + description: 'Commit message to use rather than the one from git' + required: false + rpm-test-version: + description: 'RPM version to test' + required: false + test-tag: + description: 'Test tag to use' + required: false + functional-test-distros-json: + # yamllint disable-line rule:line-length + description: 'Distros to run Functional testing on in a JSON array (i.e. ["el8", "el9", "leap15", "etc."])' + required: false + functional-test-distros: + description: 'Distros to run Functional testing on (i.e. el8 el9 leap15, etc.)' + required: false + pull_request: + push: + branches: + - master + - 'release/*' + +concurrency: + group: bullseye-coverage-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash --noprofile --norc -ueo pipefail {0} + +permissions: {} + +jobs: + Variables: + # What a dumb jobs this is + # Needed because of https://github.com/orgs/community/discussions/26671 + # Ideally want to be able to use: + # with: + # NAME: ${{ env.DISTROS }} + # in the Call-RPM-Build job but the above issue prevents it + name: Compute outputs + runs-on: [self-hosted, light] + outputs: + NAME: ${{ env.NAME }} + DISTROS: ${{ env.DISTROS }} + EL8_BUILD_VERSION: ${{ env.EL8_BUILD_VERSION }} + EL9_BUILD_VERSION: ${{ env.EL9_BUILD_VERSION }} + LEAP15_VERSION: ${{ env.LEAP15_VERSION }} + PACKAGING_DIR: ${{ env.PACKAGING_DIR }} + COVFN_DISABLED: ${{ env.COVFN_DISABLED }} + TEST_TAG: ${{ env.TEST_TAG }} + steps: + - name: Make outputs from env variables + run: echo "Make outputs from env variables" + + Call-RPM-Build: + name: Build RPM + needs: Variables + if: inputs.rpm-test-version == '' + permissions: + statuses: write + uses: daos-stack/actions-lib/.github/workflows/rpm-build.yml@bmurrell/initial + secrets: inherit + with: + NAME: ${{ needs.Variables.outputs.NAME }} + DISTROS: ${{ needs.Variables.outputs.DISTROS }} + EL8_BUILD_VERSION: ${{ needs.Variables.outputs.EL8_BUILD_VERSION }} + EL9_BUILD_VERSION: ${{ needs.Variables.outputs.EL9_BUILD_VERSION }} + LEAP15_VERSION: ${{ needs.Variables.outputs.LEAP15_VERSION }} + PACKAGING_DIR: ${{ needs.Variables.outputs.PACKAGING_DIR }} + COVFN_DISABLED: ${{ fromJSON(needs.Variables.outputs.COVFN_DISABLED) }} + + Calc-functional-matrix: + name: Calculate Functional Testing Matrix + runs-on: [self-hosted, wolf] + needs: Call-RPM-Build + if: | + (needs.Call-RPM-Build.outputs.run-gha == 'true' || + github.event_name == 'workflow_dispatch') && + ((!cancelled()) || success() || failure()) + outputs: + matrix: ${{ steps.matrix.outputs.text }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Import commit pragmas + uses: daos-stack/action-import-commit-pragmas@v1 + with: + commit-message: ${{ needs.Call-RPM-Build.outputs.dequoted-commit-message }} + - name: Calculate Functional Testing Matrix + id: matrix + run: | # do not use the non-| format for this script + . ci/gha_functions.sh + set -eu + # it might seem tempting to factor in the result of the build for this + # distro here and not include a failed build in the test matrix but + # the problem with that is that if/when the user asks GHA to rebuild + # all failed jobs and a previously failed RPM job is successful, the + # test matrix won't include testing it since it was calculated and was + # successful on the previous run without the failed build stage in it + # Use inputs.functional-test-distros-json if it was given + if [ -n '${{ inputs.functional-test-distros-json }}' ]; then + echo 'text=${{ inputs.functional-test-distros-json }}' >> $GITHUB_OUTPUT + echo 'Using JSON input parameter instead of calculating' + cat $GITHUB_OUTPUT + exit 0 + fi + l=() + trap 'echo "text=[$(IFS=","; echo "${l[*]}")]" >> $GITHUB_OUTPUT; \ + cat $GITHUB_OUTPUT' EXIT + # Use inputs.functional-test-distros if it was given + if [ -n '${{ inputs.functional-test-distros }}' ]; then + ds=(${{ inputs.functional-test-distros }}) + for d in "${ds[@]}"; do + l+=("\"$d\"") + done + echo 'Using input parameter instead of calculating' + exit 0 + fi + if ${CP_SKIP_FUNC_TEST:-false}; then + exit 0 + fi + if ! cd src/tests/ftest; then + echo "src/tests/ftest doesn't exist." + echo "Could not determine if tests exist for this stage, assuming they do." + exit 0 + fi + + if ./launch.py --list "$(get_test_tags "vm")"; then + if ! ${CP_SKIP_BUILD_EL8_RPM:-false} && + ! ${CP_SKIP_FUNC_TEST_EL8:-false}; then + # it would definitely be nicer to get these into the environment + # as unquoted strings so that we didn't have to double quote here + l+=('"el8"') + fi + if ${{ github.event_name == 'push' }} || + (${{ github.event_name == 'pull_request' }} && + ! ${CP_SKIP_BUILD_EL9_RPM:-false} && + ! ${CP_SKIP_FUNC_TEST_EL9:-true}); then + l+=('"el9"') + fi + if ${{ github.event_name == 'push' }} || + (${{ github.event_name == 'pull_request' }} && + ! ${CP_SKIP_BUILD_LEAP15_RPM:-false} && + ! ${CP_SKIP_FUNC_TEST_LEAP15:-true}); then + l+=('"leap15"') + fi + fi + + Functional: + name: Functional Testing + runs-on: [self-hosted, wolf] + permissions: + statuses: write + # https://github.com/EnricoMi/publish-unit-test-result-action#permissions + checks: write + pull-requests: write + timeout-minutes: 7200 + needs: [Calc-functional-matrix, Call-RPM-Build] + strategy: + matrix: + distro: ${{ fromJSON(needs.Calc-functional-matrix.outputs.matrix) }} + fail-fast: false + # https://github.com/actions/runner/issues/491#issuecomment-926924523 + if: | + (needs.Call-RPM-Build.outputs.run-gha == 'true' || + github.event_name == 'workflow_dispatch') && + needs.Calc-functional-matrix.outputs.matrix != '[]' && + (!cancelled()) && + (needs.Call-RPM-Build.result == 'success' || + needs.Call-RPM-Build.result == 'skipped') + env: + CONFIG_POWER_ONLY: false + PRAGMA_SUFFIX: -vm + OPERATIONS_EMAIL: brian.murrell@intel.com + TEST_RPMS: true + COMMIT_MESSAGE: ${{ needs.Call-RPM-Build.outputs.commit-message }} + JENKINS_URL: https://build.hpdd.intel.com/ + REPOSITORY_URL: https://repo.dc.hpdd.intel.com/ + REMOVE_EXISTING_RPMS: false + # TODO -- this should be on stable, backedup storage + ARTIFACTS_URL: file:///scratch/job_repos/ + REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ + # keep VS Code's GHA linting happy + NODESTRING: + CP_PR_REPOS: + CP_FEATURES: + CP_TEST_TAG: + CP_EL8_VM9_LABEL: + CP_EL9_VM9_LABEL: + CP_LEAP15_VM9_LABEL: + CP_PRIORITY: + CP_EL8_VERSION: + CP_EL9_VERSION: + CP_LEAP15_VERSION: + CP_RPM_TEST_VERSION: + DISTRO: + CLUSTER_REQUEST_reqid: + STAGE_NAME: + QUEUE_URL: + LABEL: + STAGE_TAGS: + FTEST_ARG: + DISTRO_NAME: + DISTRO_VERSION: + COMMIT_STATUS_DISTRO_VERSION: + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 500 + ref: ${{ github.event.pull_request.head.sha }} + - name: Import commit pragmas + uses: daos-stack/action-import-commit-pragmas@v1 + with: + commit-message: ${{ needs.Call-RPM-Build.outputs.dequoted-commit-message }} + - name: Set variables + run: | + set -eux + env + STAGE_TAGS="vm" + FTEST_ARG="" + TARGET=${{ github.event_name == 'pull_request' && + github.event.pull_request.base.ref || github.ref }} + echo ${{ format('{0}', github.event.pull_request.base.ref) }} + echo "NEXT_VERSION == \"${{ env.NEXT_VERSION_[format('{0}', + github.event.pull_request.base.ref)] }}\"" + TARGET="${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}" + INST_RPMS=daos-{server,client,tests{,-internal},serialize} + # TODO: replace 'master' below with the actual [target, in the case of PRs] branch name + if [ -n '${{ inputs.rpm-test-version }}' ]; then + DAOS_VERSION='-${{ inputs.rpm-test-version }}' + elif [ -n "${{ env.CP_RPM_TEST_VERSION }}" ]; then + DAOS_VERSION="-${{ env.CP_RPM_TEST_VERSION }}" + else + DAOS_VERSION=" < ${{ env.NEXT_VERSION_master }}" + fi + case '${{ matrix.distro }}' in + 'el8') + CHROOT_NAME="rocky+epel-8-x86_64" + DISTRO_NAME="EL" + DISTRO_NAME_UPPER="EL" + DISTRO_NAME_LOWER="el" + DISTRO_VERSION='${{ env.CP_EL8_VERSION && + env.CP_EL8_VERSION || env.EL8_VERSION }}' + DISTRO_VERSION_MAJOR="8" + OPENMPI="openmpi" + LABEL='${{ env.CP_EL8_VM9_LABEL && + env.CP_EL8_VM9_LABEL || 'all_vm9' }}' + ;; + 'el9') + CHROOT_NAME="rocky+epel-9-x86_64" + DISTRO_NAME="EL" + DISTRO_NAME_UPPER="EL" + DISTRO_NAME_LOWER="el" + DISTRO_VERSION='${{ env.CP_EL9_VERSION && + env.CP_EL9_VERSION || env.EL9_VERSION }}' + DISTRO_VERSION_MAJOR="9" + PROV_DISTRO_VERSION_MAJOR="8" + OPENMPI="openmpi" + LABEL='${{ env.CP_EL9_VM9_LABEL && + env.CP_EL9_VM9_LABEL || 'all_vm9' }}' + ;; + 'leap15') + CHROOT_NAME='opensuse-leap-${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || + env.LEAP15_VERSION }}-x86_64' + DISTRO_NAME="Leap" + DISTRO_NAME_UPPER="LEAP" + DISTRO_NAME_LOWER="leap" + DISTRO_VERSION='${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}' + DISTRO_VERSION_MAJOR="15" + OPENMPI="openmpi3" + LABEL='${{ env.CP_LEAP15_VM9_LABEL && + env.CP_LEAP15_VM9_LABEL || 'all_vm9' }}' + ;; + esac + echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV + echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV + echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV + echo "DISTRO_WITH_VERSION=$DISTRO_NAME_LOWER$DISTRO_VERSION" >> $GITHUB_ENV + echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME"'-${{ github.run_id }}/' >> $GITHUB_ENV + echo "STAGE_NAME=Functional on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV + echo "STAGE_TAGS=$STAGE_TAGS" >> $GITHUB_ENV + echo "FTEST_ARG=$FTEST_ARG" >> $GITHUB_ENV + echo "DISTRO=${DISTRO_NAME_UPPER}_$DISTRO_VERSION_MAJOR" >> $GITHUB_ENV + echo -n "PROVISION_DISTRO=${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV + echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}" >> $GITHUB_ENV + echo -n "DAOS_STACK_${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV + echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}_LOCAL_REPO=not_used" >> \ + $GITHUB_ENV + echo "LABEL=$LABEL" >> $GITHUB_ENV + echo "INST_RPMS=$INST_RPMS" >> $GITHUB_ENV + echo "DAOS_VERSION=$DAOS_VERSION" >> $GITHUB_ENV + - name: Request and Provision a Cluster + timeout-minutes: 7200 + uses: ./.github/actions/provision-cluster + with: + condition: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && \ + env.CP_SKIP_FUNC_TEST != 'true' + - name: Run Test + timeout-minutes: 7200 + if: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && env.CP_SKIP_FUNC_TEST != 'true' + id: run-test + run: | + . ci/gha_functions.sh + if [ -n '${{ inputs.test-tag }}' ]; then + REQ_TEST_TAG='${{ inputs.test-tag }}' + elif [ -n '${{ env.CP_TEST_TAG }}' ]; then + REQ_TEST_TAG='${{ env.CP_TEST_TAG }}' + fi + NODE_COUNT="$NODE_COUNT" \ + TEST_TAG="$(get_test_tags ${{ env.STAGE_TAGS}})" \ + FTEST_ARG='${{ env.FTEST_ARG }}' ci/functional/test_main.sh + - name: Cancel cluster request (if cancelled after requesting) + if: cancelled() + run: | + set -eux + . ci/gha_functions.sh + if ! JENKINS_URL='${{ env.JENKINS_URL }}' QUEUE_URL='${{ env.QUEUE_URL }}' \ + cancel_provision; then + # probably already provisioned and needs unprovisioning + if ! cleanup_provision_request '${{ env.CLUSTER_REQUEST_reqid }}'; then + exit 1 + fi + fi + - name: Job cleanup + if: (!cancelled() && (success() || failure())) + run: | + set -eux + . ci/gha_functions.sh + NODELIST=${{ env.NODESTRING }} ci/functional/job_cleanup.sh || true + cleanup_provision_request '${{ env.CLUSTER_REQUEST_reqid }}' + - name: Publish test results + if: (!cancelled()) && (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + check_name: ${{ env.STAGE_NAME }} Test Results + github_token: ${{ secrets.GITHUB_TOKEN }} + junit_files: ${{ env.STAGE_NAME }}/**/results.xml + - name: Publish artifacts + if: (!cancelled()) && (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} artifacts + path: ${{ env.STAGE_NAME }}/** + - name: Upload test results + if: (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} test-results + path: ${{ env.STAGE_NAME }}/**/results.xml + - name: Update commit status + if: contains(fromJSON('["push", "pull_request"]'), github.event_name) + uses: ouzi-dev/commit-status-updater@v2 + with: + # yamllint disable-line rule:line-length + name: 'test/Functional on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' + status: "${{ job.status }}" + + Calc-functional-hardware-matrix: + name: Calculate Functional Hardware Testing Matrix + runs-on: [self-hosted, wolf] + needs: Call-RPM-Build + if: | + (needs.Call-RPM-Build.outputs.run-gha == 'true' || + github.event_name == 'workflow_dispatch') && + ((!cancelled()) || success() || failure()) + env: + # keep VS Code's GHA linting happy + CP_TEST_TAG: + outputs: + matrix: ${{ steps.matrix.outputs.text }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Import commit pragmas + uses: daos-stack/action-import-commit-pragmas@v1 + with: + commit-message: ${{ needs.Call-RPM-Build.outputs.dequoted-commit-message }} + - name: Calculate Functional Testing Matrix + id: matrix + run: | # do not use the non-| format for this script + . ci/gha_functions.sh + set -eux + # it might seem tempting to factor in the result of the build for this + # distro here and not include a failed build in the test matrix but + # the problem with that is that if/when the user asks GHA to rebuild + # all faiiled jobs and a previously failed RPM job is successful, the + # test matrix won't include testing it since it was calculated and was + # successful on the previous run without the failed build stage in it + l=() + trap 'echo "text=[$(IFS=","; echo "${l[*]}")]" >> $GITHUB_OUTPUT; \ + cat $GITHUB_OUTPUT' EXIT + if ${CP_SKIP_FUNC_HW_TEST:-false}; then + exit 0 + fi + if ! cd src/tests/ftest; then + echo "src/tests/ftest doesn't exist." + echo "Could not determine if tests exist for this stage, assuming they do." + exit 0 + fi + if [ -n '${{ inputs.test-tag }}' ]; then + REQ_TEST_TAG='${{ inputs.test-tag }}' + elif [ -n '${{ env.CP_TEST_TAG }}' ]; then + REQ_TEST_TAG='${{ env.CP_TEST_TAG }}' + fi + if ! "${CP_SKIP_FUNC_HW_TEST_LARGE:-false}" && + ./launch.py --list "$(get_test_tags "hw,large,-provider")"; then + # it would definitely be nicer to get these into the environment + # as unquoted strings so that we didn't have to double quote here + l+=('"Large"') + fi + if ! ${CP_SKIP_FUNC_HW_TEST_MEDIUM:-false} && + ./launch.py --list "$(get_test_tags "hw,medium,-provider")"; then + l+=('"Medium"') + fi + if ! ${CP_SKIP_FUNC_HW_TEST_MEDIUM_VERBS_PROVIDER:-false} && + ./launch.py --list "$(get_test_tags "hw,medium,provider")"; then + l+=('"Medium Verbs Provider"') + fi + if ${{ github.event_name == 'push' }} && + ! ${CP_SKIP_FUNC_HW_TEST_MEDIUM_UCX_PROVIDER:-false} && + ./launch.py --list "$(get_test_tags "hw,medium,provider")"; then + l+=('"Medium UCX Provider"') + fi + + Functional_Hardware: + name: Functional Testing on Hardware + runs-on: [self-hosted, wolf] + permissions: + statuses: write + # https://github.com/EnricoMi/publish-unit-test-result-action#permissions + checks: write + pull-requests: write + timeout-minutes: 7200 + needs: [Calc-functional-hardware-matrix, + Call-RPM-Build, Functional] + strategy: + matrix: + stage: ${{ fromJSON(needs.Calc-functional-hardware-matrix.outputs.matrix) }} + fail-fast: false + # https://github.com/actions/runner/issues/491#issuecomment-926924523 + if: | + (needs.Call-RPM-Build.outputs.run-gha == 'true' || + github.event_name == 'workflow_dispatch') && + needs.Calc-functional-hardware-matrix.outputs.matrix != '[]' && + (!cancelled()) && + (needs.Call-RPM-Build.result == 'success' || + needs.Call-RPM-Build.result == 'skipped') && + (needs.Functional.result == 'success' || + needs.Functional.result == 'skipped') + env: + CONFIG_POWER_ONLY: false + PRAGMA_SUFFIX: -vm + OPERATIONS_EMAIL: brian.murrell@intel.com + TEST_RPMS: true + COMMIT_MESSAGE: ${{ needs.Call-RPM-Build.outputs.commit-message }} + JENKINS_URL: https://build.hpdd.intel.com/ + REPOSITORY_URL: https://repo.dc.hpdd.intel.com/ + REMOVE_EXISTING_RPMS: false + # TODO -- this should be on stable, backedup storage + ARTIFACTS_URL: file:///scratch/job_repos/ + REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ + # keep VS Code's GHA linting happy + NODESTRING: + CP_PR_REPOS: + CP_TEST_TAG: + CP_HW_MEDIUM_LABEL: + CP_HW_LARGE_LABEL: + CP_PRIORITY: + CP_EL8_VERSION: + CP_EL8_TARGET: + CLUSTER_REQUEST_reqid: + STAGE_NAME: + QUEUE_URL: + LABEL: + COMMIT_STATUS_DISTRO_VERSION: + FTEST_ARG: + STAGE_TAGS: + SIZE: + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 500 + ref: ${{ github.event.pull_request.head.sha }} + - name: Import commit pragmas + uses: daos-stack/action-import-commit-pragmas@v1 + with: + commit-message: ${{ needs.Call-RPM-Build.outputs.dequoted-commit-message }} + - name: Set variables + run: | + STAGE_TAGS="hw" + FTEST_ARG="--nvme=auto:-3DNAND" + INST_RPMS=daos-{server,client,tests{,-internal},serialize} + # TODO: replace 'master' below with the actual [target, in the case of PRs] branch name + if [ -n '${{ inputs.rpm-test-version }}' ]; then + DAOS_VERSION='-${{ inputs.rpm-test-version }}' + else + DAOS_VERSION=" < ${{ env.NEXT_VERSION_master }}" + fi + CHROOT_NAME="rocky+epel-8-x86_64" + DISTRO_NAME="EL" + DISTRO_NAME_UPPER="EL" + DISTRO_NAME_LOWER="el" + DISTRO_VERSION='${{ env.CP_EL8_TARGET && + env.CP_EL8_TARGET || + env.CP_EL8_VERSION && + env.CP_EL8_VERSION || env.EL8_VERSION }}' + DISTRO_VERSION_MAJOR="8" + if [[ '${{ matrix.stage }}' = Medium* ]]; then + LABEL=${{ env.CP_HW_MEDIUM_LABEL && + env.CP_HW_MEDIUM_LABEL || 'ci_nvme5' }} + STAGE_TAGS+=",medium" + SIZE="MEDIUM" + elif [[ '${{ matrix.stage }}' = Large* ]]; then + LABEL=${{ env.CP_HW_LARGE_LABEL && + env.CP_HW_LARGE_LABEL || 'ci_nvme9' }} + STAGE_TAGS+=",large" + SIZE="LARGE" + fi + if [[ '${{ matrix.stage }}' = *\ Provider ]]; then + STAGE_TAGS+=",provider" + if [[ '${{ matrix.stage }}' = *\ Verbs\ * ]]; then + FTEST_ARG+=' --provider ofi+verbs' + elif [[ '${{ matrix.stage }}' = *\ UCX\ * ]]; then + FTEST_ARG+=' --provider ucx+dc_x' + INST_RPMS+=' mercury-ucx' + elif [[ '${{ matrix.stage }}' = *\ TCP\ * ]]; then + FTEST_ARG+=' --provider ofi+tcp' + else + echo 'Unknown provider in ${{ matrix.stage }}' + exit 1 + fi + else + STAGE_TAGS+=",-provider" + fi + echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV + echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV + echo "DISTRO_WITH_VERSION=$DISTRO_NAME_LOWER$DISTRO_VERSION" >> $GITHUB_ENV + echo 'STAGE_NAME=Functional Hardware ${{ matrix.stage }}' >> $GITHUB_ENV + echo "STAGE_TAGS=$STAGE_TAGS" >> $GITHUB_ENV + echo "FTEST_ARG=$FTEST_ARG" >> $GITHUB_ENV + echo "DISTRO=${DISTRO_NAME_UPPER}_$DISTRO_VERSION_MAJOR" >> $GITHUB_ENV + echo -n "PROVISION_DISTRO=${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV + echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}" >> $GITHUB_ENV + echo -n "DAOS_STACK_${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV + echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}_LOCAL_REPO=not_used" >> \ + $GITHUB_ENV + echo "LABEL=$LABEL" >> $GITHUB_ENV + echo "INST_RPMS=$INST_RPMS" >> $GITHUB_ENV + echo "DAOS_VERSION=$DAOS_VERSION" >> $GITHUB_ENV + echo "SIZE=$SIZE" >> $GITHUB_ENV + - name: Request and Provision a Cluster + timeout-minutes: 7200 + uses: ./.github/actions/provision-cluster + with: + condition: env.CP_SKIP_FUNC_HW_TEST-${{ env.SIZE }} != 'true' && \ + env.CP_SKIP_FUNC_HW_TEST != 'true' + - name: Run Test + timeout-minutes: 7200 + if: env.CP_SKIP_FUNC_HW_TEST-${{ env.SIZE }} != 'true' && env.CP_SKIP_FUNC_HW_TEST != 'true' + id: run-test + run: | + . ci/gha_functions.sh + if [ -n '${{ inputs.test-tag }}' ]; then + REQ_TEST_TAG='${{ inputs.test-tag }}' + elif [ -n '${{ env.CP_TEST_TAG }}' ]; then + REQ_TEST_TAG='${{ env.CP_TEST_TAG }}' + fi + NODE_COUNT="$NODE_COUNT" \ + TEST_TAG="$(get_test_tags ${{ env.STAGE_TAGS}})" \ + FTEST_ARG='${{ env.FTEST_ARG }}' ci/functional/test_main.sh + - name: Cancel cluster request (if cancelled after requesting) + if: cancelled() + run: | + set -eux + . ci/gha_functions.sh + if ! JENKINS_URL='${{ env.JENKINS_URL }}' QUEUE_URL='${{ env.QUEUE_URL }}' \ + cancel_provision; then + # probably already provisioned and needs unprovisioning + if ! cleanup_provision_request '${{ env.CLUSTER_REQUEST_reqid }}'; then + exit 1 + fi + fi + - name: Job cleanup + if: (!cancelled() && (success() || failure())) + run: | + set -eux + . ci/gha_functions.sh + cleanup_provision_request '${{ env.CLUSTER_REQUEST_reqid }}' + NODELIST=${{ env.NODESTRING }} ci/functional/job_cleanup.sh + - name: Publish test results + if: (!cancelled()) && (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + check_name: ${{ env.STAGE_NAME }} Test Results + github_token: ${{ secrets.GITHUB_TOKEN }} + junit_files: ${{ env.STAGE_NAME }}/**/results.xml + - name: Publish artifacts + if: (!cancelled()) && (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} artifacts + path: ${{ env.STAGE_NAME }}/** + - name: Upload test results + if: (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} test-results + path: ${{ env.STAGE_NAME }}/**/results.xml + - name: Update commit status + if: contains(fromJSON('["push", "pull_request"]'), github.event_name) + uses: ouzi-dev/commit-status-updater@v2 + with: + name: 'test/Functional Hardware ${{ matrix.stage }}' + status: "${{ job.status }}" diff --git a/.github/workflows/rpm-build-and-test.yml b/.github/workflows/rpm-build-and-test.yml index a770293959f..5780d183bda 100644 --- a/.github/workflows/rpm-build-and-test.yml +++ b/.github/workflows/rpm-build-and-test.yml @@ -9,6 +9,13 @@ env: EL9_BUILD_VERSION: 9 EL9_VERSION: 9 LEAP15_VERSION: 15.5 + NEXT_VERSION_master: 1000 + # Which distros to build for + DISTROS: el8 el9 leap15 + TEST_TAG: pr + PACKAGING_DIR: utils/rpms + NAME: daos + COVFN_DISABLED: true on: workflow_dispatch: @@ -16,7 +23,27 @@ on: pr-repos: description: 'Any PR-repos that you want included in this build' required: false + commit-message: + description: 'Commit message to use rather than the one from git' + required: false + rpm-test-version: + description: 'RPM version to test' + required: false + test-tag: + description: 'Test tag to use' + required: false + functional-test-distros-json: + # yamllint disable-line rule:line-length + description: 'Distros to run Functional testing on in a JSON array (i.e. ["el8", "el9", "leap15", "etc."])' + required: false + functional-test-distros: + description: 'Distros to run Functional testing on (i.e. el8 el9 leap15, etc.)' + required: false pull_request: + push: + branches: + - master + - 'release/*' concurrency: group: rpm-build-and-test-${{ github.head_ref || github.run_id }} @@ -29,302 +56,63 @@ defaults: permissions: {} jobs: - # it's a real shame that this step is even needed. push events have the commit message # in - # ${{ github.event.head_commit.message }} but pull_requests don't. :-( - Import-commit-message: - name: Get commit message - if: github.repository == 'daos-stack/daos' - runs-on: [self-hosted, light] - # Map a step output to a job output - outputs: - message: ${{ steps.commit_message.outputs.text }} - dequoted_message: ${{ steps.dequoted_commit_message.outputs.text }} + Test-Variables: + name: Test Variables + runs-on: [self-hosted, wolf] steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Import Commit Message - id: commit_message - run: echo "text<> $GITHUB_OUTPUT; - git show -s --format=%B >> $GITHUB_OUTPUT; - echo "EOF" >> $GITHUB_OUTPUT; - - name: Import and Dequote Commit Message - id: dequoted_commit_message - run: . ci/gha_functions.sh; - echo "text<> $GITHUB_OUTPUT; - git show -s --format=%B | escape_single_quotes >> $GITHUB_OUTPUT; - echo "EOF" >> $GITHUB_OUTPUT; - - name: Identify Commit Pragmas - run: . ci/gha_functions.sh; - echo '${{steps.dequoted_commit_message.outputs.text }}' | get_commit_pragmas - - Import-commit-pragmas: - name: Make commit pragma variables - runs-on: [self-hosted, light] - needs: [Import-commit-message] - # can't use matrixes for matrixed output yet - # https://github.com/actions/runner/pull/2477 - # strategy: - # matrix: - # distro: [el8, el9, leap15] - # include: - # - distro: el8 - # UC_DISTRO: EL8 - # - distro: el9 - # UC_DISTRO: EL9 - # - distro: leap15 - # UC_DISTRO: LEAP15 - # Map a step output to a job output - outputs: - rpm-test-version: ${{ steps.rpm-test-version.outputs.value }} - pr-repos: ${{ steps.pr-repos.outputs.value }} - run-gha: ${{ steps.run-gha.outputs.value }} - steps: - - name: Set rpm-test-version variable - id: rpm-test-version - uses: ./.github/actions/variable-from-pragma - with: - commit_message: ${{ needs.Import-commit-message.outputs.dequoted_message }} - pragma: RPM_TEST_VERSION - - name: Set pr-repos variable - id: pr-repos - uses: ./.github/actions/variable-from-pragma - with: - commit_message: ${{ needs.Import-commit-message.outputs.dequoted_message }} - pragma: PR_REPOS - - name: Set run-gha variable - id: run-gha - uses: ./.github/actions/variable-from-pragma - with: - commit_message: ${{ needs.Import-commit-message.outputs.dequoted_message }} - pragma: RUN_GHA - default: false - - Create-symlinks: - # you might think this is an odd place to do this and it should be done as a result of the - # build and/or testing stages and ideally you'd be right. - # the problem with that is that there is no way to get the success/fail result of individual - # axes of matrix jobs so there is no way to query them at the end and see their composite - # results. - # instead, the final result of the Build-RPM job, for example is a last-one-complete wins. - # so for example, if the el9 axis fails quickly and then the el8 axis succeeds afterward the - # resulting job state is success. - # instead we assume success at the beginning and then let any axis that fails remove the - # lastSuccessfulBuild link if it fails - name: Create lastBuild and lastSuccessfulBuild symlinks - runs-on: [self-hosted, light] - needs: [Import-commit-pragmas] - if: needs.Import-commit-pragmas.outputs.run-gha == 'true' && - needs.Import-commit-pragmas.outputs.rpm-test-version == '' && - !contains(needs.Import-commit-pragmas.outputs.pr-repos, 'daos@') - env: - # TODO -- this should be on stable, backedup storage, not /scratch - # yamllint disable-line rule:line-length - REPO_PATH: /scratch/job_repos/daos-stack/job/daos/job/PR-${{ github.event.pull_request.number }}/ - steps: - - name: Create lastBuild and lastSuccessfulBuild symlinks - run: . ci/gha_functions.sh; - mkdir -p ${REPO_PATH}; - rm -f ${REPO_PATH}last{,Successful}Build; - ln -s ${{ github.run_number }} ${REPO_PATH}lastBuild; - ln -s ${{ github.run_number }} ${REPO_PATH}lastSuccessfulBuild - - Calc-rpm-build-matrix: - name: Calculate RPM Build Matrix - runs-on: [self-hosted, wolf] - needs: [Import-commit-pragmas, Create-symlinks] - outputs: - matrix: ${{ steps.matrix.outputs.text }} - steps: - - name: Import commit pragmas - uses: ./.github/actions/import-commit-pragmas - - name: Calculate RPM Build Matrix - id: matrix - run: | # do not use the non-| format for this script - l=() - trap 'echo "text=[$(IFS=","; echo "${l[*]}")]" >> $GITHUB_OUTPUT' EXIT - if ${CP_SKIP_BUILD:-false}; then - exit 0 - fi - if ! ${CP_SKIP_BUILD_EL8_RPM:-false}; then - l+=('"el8"') - fi - if ! ${CP_SKIP_BUILD_EL9_RPM:-false}; then - l+=('"el9"') - fi - if ${{ github.event_name == 'push' }} || - (${{ github.event_name == 'pull_request' }} && - ! ${CP_SKIP_BUILD_LEAP15_RPM:-false}); then - l+=('"leap15"') - fi + - name: Convert Varibles + id: variables + uses: ./.github/actions/variables-from-env + - run: | + set -eux + echo "NAME: ${{ steps.variables.outputs.NAME }}" + echo "DISTROS: ${{ steps.variables.outputs.DISTROS }}" + echo "EL8_BUILD_VERSION: ${{ steps.variables.outputs.EL8_BUILD_VERSION }}" + echo "EL9_BUILD_VERSION: ${{ steps.variables.outputs.EL9_BUILD_VERSION }}" + echo "LEAP15_VERSION: ${{ steps.variables.outputs.LEAP15_VERSION }}" + echo "PACKAGING_DIR: ${{ steps.variables.outputs.PACKAGING_DIR }}" + echo "COVFN_DISABLED: ${{ steps.variables.outputs.COVFN_DISABLED }}" - Build-RPM: + Call-RPM-Build: name: Build RPM + needs: Variables + if: inputs.rpm-test-version == '' permissions: statuses: write - runs-on: [self-hosted, docker] - needs: [Create-symlinks, Import-commit-pragmas, Calc-rpm-build-matrix] - if: needs.Import-commit-pragmas.outputs.run-gha == 'true' && - needs.Create-symlinks.result == 'success' && - ((!cancelled()) || success() || failure()) - strategy: - matrix: - distro: ${{ fromJSON(needs.Calc-rpm-build-matrix.outputs.matrix) }} - fail-fast: false - env: - ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ - DAOS_EMAIL: brian.murrell@intel.com - DAOS_FULLNAME: daos-stack - DISTRO: ${{ matrix.distro }} - DISTRO_REPOS: disabled - DOCKER_BUILDKIT: 0 - JENKINS_URL: https://build.hpdd.intel.com/ - ARTIFACTS_URL: file:///scratch/job_repos/ - MOCK_OPTIONS: --uniqueext=${{ github.run_id }} - PR_NUM: ${{ github.event.pull_request.number }} - # TODO -- this should be on stable, backedup storage, not /scratch - # yamllint disable-line rule:line-length - REPO_PATH: /scratch/job_repos/daos-stack/job/daos/job/PR-${{ github.event.pull_request.number }}/ - REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ - RUN_ID: ${{ github.run_id }} - TARGET: ${{ matrix.distro }} - # keep VS Code's GHA linting happy - STAGE_NAME: - DISTRO_NAME: - DISTRO_VERSION: - CP_LEAP15_VERSION: - COMMIT_STATUS_DISTRO_VERSION: - FVERSION: - steps: - - name: Import commit pragmas - uses: ./.github/actions/import-commit-pragmas - - name: Set variables - run: | - FVERSION="38" - case ${{ matrix.distro }} in - 'el8') - CHROOT_NAME="rocky+epel-8-x86_64" - DISTRO_NAME="EL" - DISTRO_VERSION="${{ env.EL8_BUILD_VERSION }}" - COMMIT_STATUS_DISTRO_VERSION="8" - ;; - 'el9') - CHROOT_NAME="rocky+epel-9-x86_64" - DISTRO_NAME="EL" - DISTRO_VERSION="${{ env.EL9_BUILD_VERSION }}" - ;; - 'leap15') - CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || - env.LEAP15_VERSION }}-x86_64" - DISTRO_NAME="Leap" - DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" - ;; - esac - echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV - echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV - echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV - echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV - echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV - echo "FVERSION=$FVERSION" >> $GITHUB_ENV - echo "COMMIT_STATUS_DISTRO_VERSION=$COMMIT_STATUS_DISTRO_VERSION" >> $GITHUB_ENV - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Build RPM Docker image - id: build-rpm-docker-image - continue-on-error: true - run: docker build --file utils/rpms/packaging/Dockerfile.mockbuild - --build-arg CACHEBUST=$(date +%s%3N) - --build-arg CB0=$(date +%V) - --build-arg REPO_FILE_URL=$REPO_FILE_URL - --build-arg UID=$(id -u) - --build-arg FVERSION=${{ env.FVERSION }} - --tag mock-build - utils/rpms - - name: Build RPM - id: build-rpm - continue-on-error: true - # yamllint disable rule:line-length - run: rm -rf mock_result; - mkdir -p mock_result; - docker run --name mock-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }} - --user build - -v "$PWD":"$PWD" -w "$PWD" - -v "$PWD"/mock_result:/var/lib/mock/$CHROOT_NAME/result - --privileged=true - -e DAOS_FULLNAME="$DAOS_FULLNAME" - -e DAOS_EMAIL="$DAOS_EMAIL" - -e DISTRO_VERSION="$DISTRO_VERSION" - -e STAGE_NAME="$STAGE_NAME" - -e CHROOT_NAME="$CHROOT_NAME" - -e ARTIFACTORY_URL="$ARTIFACTORY_URL" - -e REPO_FILE_URL="$REPO_FILE_URL" - -e JENKINS_URL="$JENKINS_URL" - -e TARGET="$TARGET" - mock-build ci/rpm/build.sh - # yamllint enable rule:line-length - - name: Build RPM failure log - id: build-rpm-fail-log - continue-on-error: true - if: steps.build-rpm.outcome != 'success' - run: cat mock_result/root.log; - cat mock_result/build.log - - name: Save RPM build logs - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: ${{ env.STAGE_NAME }} logs - path: | - mock_result/root.log - mock_result/build.log - - name: Create repo - id: create-repo - if: steps.build-rpm.outcome == 'success' - continue-on-error: true - run: CHROOT_NAME=$CHROOT_NAME ci/rpm/create_repo.sh - - name: Test repo - id: test-repo - if: steps.create-repo.outcome == 'success' - continue-on-error: true - run: . ci/gha_functions.sh; - dnf --disablerepo=\* --repofrompath - testrepo,file://${REPO_PATH}${{ github.run_number }}/artifact/artifacts/$TARGET - repoquery -a - - name: Remove lastSuccessfulBuild link and exit failure - if: steps.test-repo.outcome != 'success' - run: rm -f ${REPO_PATH}lastSuccessfulBuild; - exit 1 - - name: Publish RPMs - uses: actions/upload-artifact@v4 - with: - name: ${{ env.DISTRO_NAME }} ${{ env.DISTRO_VERSION }} RPM repository - path: ${{ env.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ env.TARGET }} - - name: Update commit status - uses: ouzi-dev/commit-status-updater@v2 - with: - # yamllint disable-line rule:line-length - name: 'build/Build RPM on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' - status: "${{ job.status }}" + uses: daos-stack/actions-lib/.github/workflows/rpm-build.yml@bmurrell/initial + secrets: inherit + with: + NAME: ${{ needs.Variables.outputs.NAME }} + DISTROS: ${{ needs.Variables.outputs.DISTROS }} + EL8_BUILD_VERSION: ${{ needs.Variables.outputs.EL8_BUILD_VERSION }} + EL9_BUILD_VERSION: ${{ needs.Variables.outputs.EL9_BUILD_VERSION }} + LEAP15_VERSION: ${{ needs.Variables.outputs.LEAP15_VERSION }} + PACKAGING_DIR: ${{ needs.Variables.outputs.PACKAGING_DIR }} + COVFN_DISABLED: ${{ fromJSON(needs.Variables.outputs.COVFN_DISABLED) }} Calc-functional-matrix: name: Calculate Functional Testing Matrix runs-on: [self-hosted, wolf] - needs: [Import-commit-pragmas] - if: needs.Import-commit-pragmas.outputs.run-gha == 'true' && + needs: Call-RPM-Build + if: | + (needs.Call-RPM-Build.outputs.run-gha == 'true' || + github.event_name == 'workflow_dispatch') && ((!cancelled()) || success() || failure()) outputs: matrix: ${{ steps.matrix.outputs.text }} steps: - - name: Import commit pragmas - uses: ./.github/actions/import-commit-pragmas - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Import commit pragmas + uses: daos-stack/action-import-commit-pragmas@v1 + with: + commit-message: ${{ needs.Call-RPM-Build.outputs.dequoted-commit-message }} - name: Calculate Functional Testing Matrix id: matrix run: | # do not use the non-| format for this script @@ -336,8 +124,25 @@ jobs: # all failed jobs and a previously failed RPM job is successful, the # test matrix won't include testing it since it was calculated and was # successful on the previous run without the failed build stage in it + # Use inputs.functional-test-distros-json if it was given + if [ -n '${{ inputs.functional-test-distros-json }}' ]; then + echo 'text=${{ inputs.functional-test-distros-json }}' >> $GITHUB_OUTPUT + echo 'Using JSON input parameter instead of calculating' + cat $GITHUB_OUTPUT + exit 0 + fi l=() - trap 'echo "text=[$(IFS=","; echo "${l[*]}")]" >> $GITHUB_OUTPUT' EXIT + trap 'echo "text=[$(IFS=","; echo "${l[*]}")]" >> $GITHUB_OUTPUT; \ + cat $GITHUB_OUTPUT' EXIT + # Use inputs.functional-test-distros if it was given + if [ -n '${{ inputs.functional-test-distros }}' ]; then + ds=(${{ inputs.functional-test-distros }}) + for d in "${ds[@]}"; do + l+=("\"$d\"") + done + echo 'Using input parameter instead of calculating' + exit 0 + fi if ${CP_SKIP_FUNC_TEST:-false}; then exit 0 fi @@ -347,15 +152,17 @@ jobs: exit 0 fi - if ./launch.py --list "$(get_test_tags "-hw")"; then + if ./launch.py --list "$(get_test_tags "vm")"; then if ! ${CP_SKIP_BUILD_EL8_RPM:-false} && ! ${CP_SKIP_FUNC_TEST_EL8:-false}; then # it would definitely be nicer to get these into the environment # as unquoted strings so that we didn't have to double quote here l+=('"el8"') fi - if ! ${CP_SKIP_BUILD_EL9_RPM:-false} && - ! ${CP_SKIP_FUNC_TEST_EL9:-false}; then + if ${{ github.event_name == 'push' }} || + (${{ github.event_name == 'pull_request' }} && + ! ${CP_SKIP_BUILD_EL9_RPM:-false} && + ! ${CP_SKIP_FUNC_TEST_EL9:-true}); then l+=('"el9"') fi if ${{ github.event_name == 'push' }} || @@ -375,24 +182,25 @@ jobs: checks: write pull-requests: write timeout-minutes: 7200 - needs: [Build-RPM, Import-commit-message, Calc-functional-matrix, Import-commit-pragmas] + needs: [Calc-functional-matrix, Call-RPM-Build] strategy: matrix: distro: ${{ fromJSON(needs.Calc-functional-matrix.outputs.matrix) }} fail-fast: false # https://github.com/actions/runner/issues/491#issuecomment-926924523 if: | - needs.Import-commit-pragmas.outputs.run-gha == 'true' && - needs.Calc-functional-matrix.outputs.matrix != '[]' && - (!cancelled()) && - (needs.Build-RPM.result == 'success' || - needs.Build-RPM.result == 'skipped') + (needs.Call-RPM-Build.outputs.run-gha == 'true' || + github.event_name == 'workflow_dispatch') && + needs.Calc-functional-matrix.outputs.matrix != '[]' && + (!cancelled()) && + (needs.Call-RPM-Build.result == 'success' || + needs.Call-RPM-Build.result == 'skipped') env: CONFIG_POWER_ONLY: false PRAGMA_SUFFIX: -vm OPERATIONS_EMAIL: brian.murrell@intel.com TEST_RPMS: true - COMMIT_MESSAGE: ${{ needs.Import-commit-message.outputs.message }} + COMMIT_MESSAGE: ${{ needs.Call-RPM-Build.outputs.commit-message }} JENKINS_URL: https://build.hpdd.intel.com/ REPOSITORY_URL: https://repo.dc.hpdd.intel.com/ REMOVE_EXISTING_RPMS: false @@ -411,70 +219,95 @@ jobs: CP_EL8_VERSION: CP_EL9_VERSION: CP_LEAP15_VERSION: + CP_RPM_TEST_VERSION: DISTRO: CLUSTER_REQUEST_reqid: STAGE_NAME: QUEUE_URL: LABEL: + STAGE_TAGS: + FTEST_ARG: DISTRO_NAME: DISTRO_VERSION: COMMIT_STATUS_DISTRO_VERSION: steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 500 + ref: ${{ github.event.pull_request.head.sha }} - name: Import commit pragmas - uses: ./.github/actions/import-commit-pragmas + uses: daos-stack/action-import-commit-pragmas@v1 + with: + commit-message: ${{ needs.Call-RPM-Build.outputs.dequoted-commit-message }} - name: Set variables run: | set -eux env - STAGE_TAGS="-hw" + STAGE_TAGS="vm" FTEST_ARG="" - INST_RPMS="daos-client daos-tests daos-server daos-serialize daos-tests-internal" - case "${{ matrix.distro }}" in + TARGET=${{ github.event_name == 'pull_request' && + github.event.pull_request.base.ref || github.ref }} + echo ${{ format('{0}', github.event.pull_request.base.ref) }} + echo "NEXT_VERSION == \"${{ env.NEXT_VERSION_[format('{0}', + github.event.pull_request.base.ref)] }}\"" + TARGET="${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}" + INST_RPMS=daos-{server,client,tests{,-internal},serialize} + # TODO: replace 'master' below with the actual [target, in the case of PRs] branch name + if [ -n '${{ inputs.rpm-test-version }}' ]; then + DAOS_VERSION='-${{ inputs.rpm-test-version }}' + elif [ -n "${{ env.CP_RPM_TEST_VERSION }}" ]; then + DAOS_VERSION="-${{ env.CP_RPM_TEST_VERSION }}" + else + DAOS_VERSION=" < ${{ env.NEXT_VERSION_master }}" + fi + case '${{ matrix.distro }}' in 'el8') CHROOT_NAME="rocky+epel-8-x86_64" DISTRO_NAME="EL" DISTRO_NAME_UPPER="EL" DISTRO_NAME_LOWER="el" - DISTRO_VERSION="${{ env.CP_EL8_VERSION && - env.CP_EL8_VERSION || env.EL8_VERSION }}" + DISTRO_VERSION='${{ env.CP_EL8_VERSION && + env.CP_EL8_VERSION || env.EL8_VERSION }}' DISTRO_VERSION_MAJOR="8" OPENMPI="openmpi" - LABEL="${{ env.CP_EL8_VM9_LABEL && - env.CP_EL8_VM9_LABEL || 'ci_vm9' }}" + LABEL='${{ env.CP_EL8_VM9_LABEL && + env.CP_EL8_VM9_LABEL || 'all_vm9' }}' ;; 'el9') CHROOT_NAME="rocky+epel-9-x86_64" DISTRO_NAME="EL" DISTRO_NAME_UPPER="EL" DISTRO_NAME_LOWER="el" - DISTRO_VERSION="${{ env.CP_EL9_VERSION && - env.CP_EL9_VERSION || env.EL9_VERSION }}" + DISTRO_VERSION='${{ env.CP_EL9_VERSION && + env.CP_EL9_VERSION || env.EL9_VERSION }}' DISTRO_VERSION_MAJOR="9" PROV_DISTRO_VERSION_MAJOR="8" OPENMPI="openmpi" - LABEL="${{ env.CP_EL9_VM9_LABEL && - env.CP_EL9_VM9_LABEL || 'ci_vm9' }}" + LABEL='${{ env.CP_EL9_VM9_LABEL && + env.CP_EL9_VM9_LABEL || 'all_vm9' }}' ;; 'leap15') - CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && + CHROOT_NAME='opensuse-leap-${{ env.CP_LEAP15_VERSION && env.CP_LEAP15_VERSION || - env.LEAP15_VERSION }}-x86_64" + env.LEAP15_VERSION }}-x86_64' DISTRO_NAME="Leap" DISTRO_NAME_UPPER="LEAP" DISTRO_NAME_LOWER="leap" - DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" + DISTRO_VERSION='${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}' DISTRO_VERSION_MAJOR="15" OPENMPI="openmpi3" - LABEL="${{ env.CP_LEAP15_VM9_LABEL && - env.CP_LEAP15_VM9_LABEL || 'ci_vm9' }}" + LABEL='${{ env.CP_LEAP15_VM9_LABEL && + env.CP_LEAP15_VM9_LABEL || 'all_vm9' }}' ;; esac echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV echo "DISTRO_WITH_VERSION=$DISTRO_NAME_LOWER$DISTRO_VERSION" >> $GITHUB_ENV - echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV + echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME"'-${{ github.run_id }}/' >> $GITHUB_ENV echo "STAGE_NAME=Functional on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV echo "STAGE_TAGS=$STAGE_TAGS" >> $GITHUB_ENV echo "FTEST_ARG=$FTEST_ARG" >> $GITHUB_ENV @@ -486,12 +319,7 @@ jobs: $GITHUB_ENV echo "LABEL=$LABEL" >> $GITHUB_ENV echo "INST_RPMS=$INST_RPMS" >> $GITHUB_ENV - - name: Checkout code - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 500 - ref: ${{ github.event.pull_request.head.sha }} + echo "DAOS_VERSION=$DAOS_VERSION" >> $GITHUB_ENV - name: Request and Provision a Cluster timeout-minutes: 7200 uses: ./.github/actions/provision-cluster @@ -504,18 +332,23 @@ jobs: id: run-test run: | . ci/gha_functions.sh + if [ -n '${{ inputs.test-tag }}' ]; then + REQ_TEST_TAG='${{ inputs.test-tag }}' + elif [ -n '${{ env.CP_TEST_TAG }}' ]; then + REQ_TEST_TAG='${{ env.CP_TEST_TAG }}' + fi NODE_COUNT="$NODE_COUNT" \ TEST_TAG="$(get_test_tags ${{ env.STAGE_TAGS}})" \ - FTEST_ARG="${{ env.FTEST_ARG }}" ci/functional/test_main.sh + FTEST_ARG='${{ env.FTEST_ARG }}' ci/functional/test_main.sh - name: Cancel cluster request (if cancelled after requesting) if: cancelled() run: | set -eux . ci/gha_functions.sh - if ! JENKINS_URL="${{ env.JENKINS_URL }}" QUEUE_URL="${{ env.QUEUE_URL }}" \ + if ! JENKINS_URL='${{ env.JENKINS_URL }}' QUEUE_URL='${{ env.QUEUE_URL }}' \ cancel_provision; then # probably already provisioned and needs unprovisioning - if ! cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}"; then + if ! cleanup_provision_request '${{ env.CLUSTER_REQUEST_reqid }}'; then exit 1 fi fi @@ -525,13 +358,13 @@ jobs: set -eux . ci/gha_functions.sh NODELIST=${{ env.NODESTRING }} ci/functional/job_cleanup.sh || true - cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}" + cleanup_provision_request '${{ env.CLUSTER_REQUEST_reqid }}' - name: Publish test results if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' uses: EnricoMi/publish-unit-test-result-action@v2 with: - check_name: ${{ env.STAGE_NAME }} Test Results (old) + check_name: ${{ env.STAGE_NAME }} Test Results github_token: ${{ secrets.GITHUB_TOKEN }} junit_files: ${{ env.STAGE_NAME }}/**/results.xml - name: Publish artifacts @@ -549,6 +382,7 @@ jobs: name: ${{ env.STAGE_NAME }} test-results path: ${{ env.STAGE_NAME }}/**/results.xml - name: Update commit status + if: contains(fromJSON('["push", "pull_request"]'), github.event_name) uses: ouzi-dev/commit-status-updater@v2 with: # yamllint disable-line rule:line-length @@ -558,23 +392,30 @@ jobs: Calc-functional-hardware-matrix: name: Calculate Functional Hardware Testing Matrix runs-on: [self-hosted, wolf] - needs: [Import-commit-pragmas] - if: needs.Import-commit-pragmas.outputs.run-gha == 'true' && + needs: Call-RPM-Build + if: | + (needs.Call-RPM-Build.outputs.run-gha == 'true' || + github.event_name == 'workflow_dispatch') && ((!cancelled()) || success() || failure()) + env: + # keep VS Code's GHA linting happy + CP_TEST_TAG: outputs: matrix: ${{ steps.matrix.outputs.text }} steps: - - name: Import commit pragmas - uses: ./.github/actions/import-commit-pragmas - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Import commit pragmas + uses: daos-stack/action-import-commit-pragmas@v1 + with: + commit-message: ${{ needs.Call-RPM-Build.outputs.dequoted-commit-message }} - name: Calculate Functional Testing Matrix id: matrix run: | # do not use the non-| format for this script . ci/gha_functions.sh - set -eu + set -eux # it might seem tempting to factor in the result of the build for this # distro here and not include a failed build in the test matrix but # the problem with that is that if/when the user asks GHA to rebuild @@ -582,7 +423,8 @@ jobs: # test matrix won't include testing it since it was calculated and was # successful on the previous run without the failed build stage in it l=() - trap 'echo "text=[$(IFS=","; echo "${l[*]}")]" >> $GITHUB_OUTPUT' EXIT + trap 'echo "text=[$(IFS=","; echo "${l[*]}")]" >> $GITHUB_OUTPUT; \ + cat $GITHUB_OUTPUT' EXIT if ${CP_SKIP_FUNC_HW_TEST:-false}; then exit 0 fi @@ -591,6 +433,11 @@ jobs: echo "Could not determine if tests exist for this stage, assuming they do." exit 0 fi + if [ -n '${{ inputs.test-tag }}' ]; then + REQ_TEST_TAG='${{ inputs.test-tag }}' + elif [ -n '${{ env.CP_TEST_TAG }}' ]; then + REQ_TEST_TAG='${{ env.CP_TEST_TAG }}' + fi if ! "${CP_SKIP_FUNC_HW_TEST_LARGE:-false}" && ./launch.py --list "$(get_test_tags "hw,large,-provider")"; then # it would definitely be nicer to get these into the environment @@ -620,27 +467,28 @@ jobs: checks: write pull-requests: write timeout-minutes: 7200 - needs: [Import-commit-message, Build-RPM, Calc-functional-hardware-matrix, - Import-commit-pragmas, Functional] + needs: [Calc-functional-hardware-matrix, + Call-RPM-Build, Functional] strategy: matrix: stage: ${{ fromJSON(needs.Calc-functional-hardware-matrix.outputs.matrix) }} fail-fast: false # https://github.com/actions/runner/issues/491#issuecomment-926924523 if: | - needs.Import-commit-pragmas.outputs.run-gha == 'true' && - needs.Calc-functional-hardware-matrix.outputs.matrix != '[]' && - (!cancelled()) && - (needs.Build-RPM.result == 'success' || - needs.Build-RPM.result == 'skipped') && - (needs.Functional.result == 'success' || - needs.Functional.result == 'skipped') + (needs.Call-RPM-Build.outputs.run-gha == 'true' || + github.event_name == 'workflow_dispatch') && + needs.Calc-functional-hardware-matrix.outputs.matrix != '[]' && + (!cancelled()) && + (needs.Call-RPM-Build.result == 'success' || + needs.Call-RPM-Build.result == 'skipped') && + (needs.Functional.result == 'success' || + needs.Functional.result == 'skipped') env: CONFIG_POWER_ONLY: false PRAGMA_SUFFIX: -vm OPERATIONS_EMAIL: brian.murrell@intel.com TEST_RPMS: true - COMMIT_MESSAGE: ${{ needs.Import-commit-message.outputs.message }} + COMMIT_MESSAGE: ${{ needs.Call-RPM-Build.outputs.commit-message }} JENKINS_URL: https://build.hpdd.intel.com/ REPOSITORY_URL: https://repo.dc.hpdd.intel.com/ REMOVE_EXISTING_RPMS: false @@ -661,45 +509,62 @@ jobs: QUEUE_URL: LABEL: COMMIT_STATUS_DISTRO_VERSION: + FTEST_ARG: + STAGE_TAGS: + SIZE: steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 500 + ref: ${{ github.event.pull_request.head.sha }} - name: Import commit pragmas - uses: ./.github/actions/import-commit-pragmas + uses: daos-stack/action-import-commit-pragmas@v1 + with: + commit-message: ${{ needs.Call-RPM-Build.outputs.dequoted-commit-message }} - name: Set variables run: | STAGE_TAGS="hw" FTEST_ARG="--nvme=auto:-3DNAND" - INST_RPMS="daos-client daos-tests daos-server daos-serialize daos-tests-internal" + INST_RPMS=daos-{server,client,tests{,-internal},serialize} + # TODO: replace 'master' below with the actual [target, in the case of PRs] branch name + if [ -n '${{ inputs.rpm-test-version }}' ]; then + DAOS_VERSION='-${{ inputs.rpm-test-version }}' + else + DAOS_VERSION=" < ${{ env.NEXT_VERSION_master }}" + fi CHROOT_NAME="rocky+epel-8-x86_64" DISTRO_NAME="EL" DISTRO_NAME_UPPER="EL" DISTRO_NAME_LOWER="el" - DISTRO_VERSION="${{ env.CP_EL8_TARGET && + DISTRO_VERSION='${{ env.CP_EL8_TARGET && env.CP_EL8_TARGET || env.CP_EL8_VERSION && - env.CP_EL8_VERSION || env.EL8_VERSION }}" + env.CP_EL8_VERSION || env.EL8_VERSION }}' DISTRO_VERSION_MAJOR="8" - if [[ "${{ matrix.stage }}" = Medium* ]]; then + if [[ '${{ matrix.stage }}' = Medium* ]]; then LABEL=${{ env.CP_HW_MEDIUM_LABEL && env.CP_HW_MEDIUM_LABEL || 'ci_nvme5' }} STAGE_TAGS+=",medium" SIZE="MEDIUM" - elif [[ "${{ matrix.stage }}" = Large* ]]; then + elif [[ '${{ matrix.stage }}' = Large* ]]; then LABEL=${{ env.CP_HW_LARGE_LABEL && env.CP_HW_LARGE_LABEL || 'ci_nvme9' }} STAGE_TAGS+=",large" SIZE="LARGE" fi - if [[ "${{ matrix.stage }}" = *\ Provider ]]; then + if [[ '${{ matrix.stage }}' = *\ Provider ]]; then STAGE_TAGS+=",provider" - if [[ "${{ matrix.stage }}" = *\ Verbs\ * ]]; then + if [[ '${{ matrix.stage }}' = *\ Verbs\ * ]]; then FTEST_ARG+=' --provider ofi+verbs' - elif [[ "${{ matrix.stage }}" = *\ UCX\ * ]]; then + elif [[ '${{ matrix.stage }}' = *\ UCX\ * ]]; then FTEST_ARG+=' --provider ucx+dc_x' INST_RPMS+=' mercury-ucx' - elif [[ "${{ matrix.stage }}" = *\ TCP\ * ]]; then + elif [[ '${{ matrix.stage }}' = *\ TCP\ * ]]; then FTEST_ARG+=' --provider ofi+tcp' else - echo "Unknown provider in ${{ matrix.stage }}" + echo 'Unknown provider in ${{ matrix.stage }}' exit 1 fi else @@ -708,7 +573,7 @@ jobs: echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV echo "DISTRO_WITH_VERSION=$DISTRO_NAME_LOWER$DISTRO_VERSION" >> $GITHUB_ENV - echo "STAGE_NAME=Functional Hardware ${{ matrix.stage }}" >> $GITHUB_ENV + echo 'STAGE_NAME=Functional Hardware ${{ matrix.stage }}' >> $GITHUB_ENV echo "STAGE_TAGS=$STAGE_TAGS" >> $GITHUB_ENV echo "FTEST_ARG=$FTEST_ARG" >> $GITHUB_ENV echo "DISTRO=${DISTRO_NAME_UPPER}_$DISTRO_VERSION_MAJOR" >> $GITHUB_ENV @@ -719,13 +584,8 @@ jobs: $GITHUB_ENV echo "LABEL=$LABEL" >> $GITHUB_ENV echo "INST_RPMS=$INST_RPMS" >> $GITHUB_ENV + echo "DAOS_VERSION=$DAOS_VERSION" >> $GITHUB_ENV echo "SIZE=$SIZE" >> $GITHUB_ENV - - name: Checkout code - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 500 - ref: ${{ github.event.pull_request.head.sha }} - name: Request and Provision a Cluster timeout-minutes: 7200 uses: ./.github/actions/provision-cluster @@ -738,18 +598,23 @@ jobs: id: run-test run: | . ci/gha_functions.sh + if [ -n '${{ inputs.test-tag }}' ]; then + REQ_TEST_TAG='${{ inputs.test-tag }}' + elif [ -n '${{ env.CP_TEST_TAG }}' ]; then + REQ_TEST_TAG='${{ env.CP_TEST_TAG }}' + fi NODE_COUNT="$NODE_COUNT" \ TEST_TAG="$(get_test_tags ${{ env.STAGE_TAGS}})" \ - FTEST_ARG="${{ env.FTEST_ARG }}" ci/functional/test_main.sh + FTEST_ARG='${{ env.FTEST_ARG }}' ci/functional/test_main.sh - name: Cancel cluster request (if cancelled after requesting) if: cancelled() run: | set -eux . ci/gha_functions.sh - if ! JENKINS_URL="${{ env.JENKINS_URL }}" QUEUE_URL="${{ env.QUEUE_URL }}" \ + if ! JENKINS_URL='${{ env.JENKINS_URL }}' QUEUE_URL='${{ env.QUEUE_URL }}' \ cancel_provision; then # probably already provisioned and needs unprovisioning - if ! cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}"; then + if ! cleanup_provision_request '${{ env.CLUSTER_REQUEST_reqid }}'; then exit 1 fi fi @@ -758,14 +623,14 @@ jobs: run: | set -eux . ci/gha_functions.sh - cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}" + cleanup_provision_request '${{ env.CLUSTER_REQUEST_reqid }}' NODELIST=${{ env.NODESTRING }} ci/functional/job_cleanup.sh - name: Publish test results if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' uses: EnricoMi/publish-unit-test-result-action@v2 with: - check_name: ${{ env.STAGE_NAME }} Test Results (old) + check_name: ${{ env.STAGE_NAME }} Test Results github_token: ${{ secrets.GITHUB_TOKEN }} junit_files: ${{ env.STAGE_NAME }}/**/results.xml - name: Publish artifacts @@ -783,6 +648,7 @@ jobs: name: ${{ env.STAGE_NAME }} test-results path: ${{ env.STAGE_NAME }}/**/results.xml - name: Update commit status + if: contains(fromJSON('["push", "pull_request"]'), github.event_name) uses: ouzi-dev/commit-status-updater@v2 with: name: 'test/Functional Hardware ${{ matrix.stage }}' diff --git a/.gitignore b/.gitignore index ceca6ac3a34..0960ff64e30 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,4 @@ utils/githooks/*.d/*-user-* .pre-commit .vscode src/include/daos_version.h +utils/rpms/mock.cfg diff --git a/ci/gha_functions.sh b/ci/gha_functions.sh index 2d93137c775..962ed221597 100644 --- a/ci/gha_functions.sh +++ b/ci/gha_functions.sh @@ -6,15 +6,6 @@ error_exit() { exit 1 } -get_repo_path() { - # shellcheck disable=SC2153 - local repo_path="$REPO_PATH$GITHUB_RUN_NUMBER/artifact/artifacts/" - mkdir -p "$repo_path" - - echo "$repo_path" - -} - cleanup_provision_request () { local reqid="$1" local file="/scratch/Get a cluster/$reqid" @@ -58,8 +49,8 @@ get_test_tags() { local test_tags=() local tags # Test-tag: has higher priority - if [ -n "${CP_TEST_TAG:-}" ]; then - tags="$CP_TEST_TAG" + if [ -n "${REQ_TEST_TAG:-}" ]; then + tags="$REQ_TEST_TAG" else tags="pr" if [ -n "${CP_FEATURES:-}" ]; then @@ -76,14 +67,6 @@ get_test_tags() { } -get_commit_pragmas() { - sed -Ene 's/^([-[:alnum:]]+): *([-\._ [:alnum:]]+)$/\1 \2/p' | while read -r a b; do - echo -n "${a//-/_}" | tr '[:lower:]' '[:upper:]' - # escape special characters in the value - echo "=$b" | sed -e 's/\([<> ]\)/\\\1/g' - done -} - wait_nodes_ready() { local wait_seconds=600 @@ -104,10 +87,6 @@ wait_nodes_ready() { exit 1 } -escape_single_quotes() { - sed -e "s/'/'\"'\"'/g" -} - # Persist these across calls cookiejar="" crumb="" @@ -243,52 +222,17 @@ Stage Name: $stage_name\" > /root/job_info # I.e. ../bash_unit/bash_unit ci/gha_functions.sh test_test_tag_and_features() { # Simple Test-tag: test - assert_equals "$(CP_TEST_TAG="always_passes always_fails" get_test_tags "-hw")" "always_passes,-hw always_fails,-hw" + assert_equals "$(REQ_TEST_TAG="always_passes always_fails" get_test_tags "-hw")" "always_passes,-hw always_fails,-hw" # Simple Features: test (no Test-tag:) assert_equals "$(CP_FEATURES="always_passes" get_test_tags "-hw")" \ "pr,-hw daily_regression,always_passes,-hw full_regression,always_passes,-hw" assert_equals "$(CP_FEATURES="foo bar" get_test_tags "-hw")" \ "pr,-hw daily_regression,foo,-hw full_regression,foo,-hw daily_regression,bar,-hw full_regression,bar,-hw" # Features: and Test-tag: - assert_equals "$(CP_TEST_TAG="always_passes always_fails" + assert_equals "$(REQ_TEST_TAG="always_passes always_fails" CP_FEATURES="foo bar" get_test_tags "-hw")" "always_passes,-hw always_fails,-hw" } -test_get_commit_pragmas() { - local msg='Escape spaces also - -'"'"'Will-not-be-a-pragma: false'"'"' should not be considered a commit -pragma, but: -Should-not-be-a-pragma: bar will be because it was not quoted. - -Skip-func-test-leap15: false -RPM-test-version: 2.5.100-13.10036.g65926e32 -Skip-PR-comments: true -Test-tag: always_passes always_fails -EL8-VM9-label: all_vm9 -EL9-VM9-label: all_vm9 -Leap15-VM9-label: all_vm9 -HW-medium-label: new_icx5 -HW-large-label: new_icx9 - -Required-githooks: true - -Signed-off-by: Brian J. Murrell -' - assert_equals "$(echo "$msg" | get_commit_pragmas)" 'SHOULD_NOT_BE_A_PRAGMA=bar\ will\ be\ because\ it\ was\ not\ quoted. -SKIP_FUNC_TEST_LEAP15=false -RPM_TEST_VERSION=2.5.100-13.10036.g65926e32 -SKIP_PR_COMMENTS=true -TEST_TAG=always_passes\ always_fails -EL8_VM9_LABEL=all_vm9 -EL9_VM9_LABEL=all_vm9 -LEAP15_VM9_LABEL=all_vm9 -HW_MEDIUM_LABEL=new_icx5 -HW_LARGE_LABEL=new_icx9 -REQUIRED_GITHOOKS=true' - -} - test_jenkins_curl() { JENKINS_URL="${JENKINS_URL:-https://build.hpdd.intel.com/}" assert_equals "$(QUIET=true VERBOSE=false jenkins_curl -X POST "${JENKINS_URL}api/xml" 3>&1 >/dev/null | tr -d '\r' | grep '^X-Content-Type-Options:')" "X-Content-Type-Options: nosniff" diff --git a/ci/provisioning/post_provision_config.sh b/ci/provisioning/post_provision_config.sh index 13f78358a83..263da7c69ed 100755 --- a/ci/provisioning/post_provision_config.sh +++ b/ci/provisioning/post_provision_config.sh @@ -57,9 +57,11 @@ if ! retry_cmd 2400 clush -B -S -l root -w "$NODESTRING" \ BRANCH_NAME=\"${BRANCH_NAME:-}\" CHANGE_TARGET=\"${CHANGE_TARGET:-}\" CI_RPM_TEST_VERSION=\"${CI_RPM_TEST_VERSION:-}\" + DAOS_VERSION=\"${DAOS_VERSION:-}\" CI_PR_REPOS=\"${CI_PR_REPOS:-}\" REPO_PATH=\"${REPO_PATH:-}\" ARTIFACTS_URL=\"${ARTIFACTS_URL:-}\" + COVFN_DISABLED=\"${COVFN_DISABLED:-true}\" $(cat ci/stacktrace.sh) $(cat ci/junit.sh) $(cat ci/provisioning/post_provision_config_common_functions.sh) diff --git a/ci/provisioning/post_provision_config_common_functions.sh b/ci/provisioning/post_provision_config_common_functions.sh index 1313a850407..38bfb5b6789 100755 --- a/ci/provisioning/post_provision_config_common_functions.sh +++ b/ci/provisioning/post_provision_config_common_functions.sh @@ -370,7 +370,11 @@ post_provision_config_nodes() { branch="${branch%:*}" fi fi - local repo_url="${ARTIFACTS_URL:-${JENKINS_URL}job/}"daos-stack/job/"$repo"/job/"${branch//\//%252F}"/"$build_number"/artifact/artifacts/$DISTRO_NAME/ + local subdir + if ! $COVFN_DISABLED; then + subdir="bullseye/" + fi + local repo_url="${ARTIFACTS_URL:-${JENKINS_URL}job/}"daos-stack/job/"$repo"/job/"${branch//\//%252F}"/"$build_number"/artifact/artifacts/"${subdir:-}"$DISTRO_NAME/ dnf -y config-manager --add-repo="$repo_url" repo="$(url_to_repo "$repo_url")" # PR-repos: should always be able to upgrade modular packages @@ -404,7 +408,7 @@ post_provision_config_nodes() { # shellcheck disable=SC2001 if [ ${#inst_rpms[@]} -gt 0 ]; then - if ! retry_dnf 360 install "${inst_rpms[@]}"; then + if ! retry_dnf 360 install "${inst_rpms[@]/%/${DAOS_VERSION:-}}"; then rc=${PIPESTATUS[0]} dump_repos return "$rc" diff --git a/utils/rpms/packaging/Makefile_packaging.mk b/utils/rpms/packaging/Makefile_packaging.mk index 65ccdba11bb..75038c1178f 100644 --- a/utils/rpms/packaging/Makefile_packaging.mk +++ b/utils/rpms/packaging/Makefile_packaging.mk @@ -428,6 +428,8 @@ packaging_check: --exclude libfabric.spec \ --exclude Makefile \ --exclude README.md \ + --exclude SECURITY.md \ + --exclude LICENSE \ --exclude _topdir \ --exclude \*.tar.\* \ --exclude \*.code-workspace \ diff --git a/utils/rpms/packaging/rpm_chrootbuild b/utils/rpms/packaging/rpm_chrootbuild index 4dcdaa4b95e..d6443b7357e 100755 --- a/utils/rpms/packaging/rpm_chrootbuild +++ b/utils/rpms/packaging/rpm_chrootbuild @@ -109,7 +109,7 @@ if [ -n "$DISTRO_VERSION" ]; then releasever_opt=("--config-opts=releasever=$DISTRO_VERSION") fi -bs_dir=/scratch/mock/cache/"${CHROOT_NAME}"-bootstrap +bs_dir=/scratch/mock/cache/"${CHROOT_NAME}"-bootstrap-$(id -u) if ls -l "$bs_dir"/root_cache/cache.tar.gz; then mkdir -p "/var/cache/mock/${CHROOT_NAME}-bootstrap/" flock "$bs_dir" -c "cp -a $bs_dir/root_cache /var/cache/mock/${CHROOT_NAME}-bootstrap" @@ -129,10 +129,16 @@ fi # Save the ccache if [ -d /scratch/ ]; then mkdir -p "$bs_dir"/ - flock "$bs_dir" -c "tar -czf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz /var/cache/mock/${CHROOT_NAME}/ccache" + if ! flock "$bs_dir" -c "tar -czf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz /var/cache/mock/${CHROOT_NAME}/ccache"; then + echo "Failed to save ccache. Plowing onward." + echo "I am $(id)" + fi if ls -l /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz; then if ! cmp /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz "$bs_dir"/root_cache/cache.tar.gz; then - flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/" + if ! flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/"; then + echo "Failed to save root_cache. Plowing onward." + echo "I am $(id)" + fi fi fi fi