diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..87b2b2b --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Have Actions Watchers added as a reviewer to any GitHub Actions +* @daos-stack/actions-watchers diff --git a/.github/workflows/bash_unit.yml b/.github/workflows/bash_unit.yml new file mode 100644 index 0000000..6cb8e17 --- /dev/null +++ b/.github/workflows/bash_unit.yml @@ -0,0 +1,18 @@ +name: bash_unit Tests +on: + push: + branches: + - master + pull_request: + +jobs: + bash-unit-testing: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Unit testing with bash_unit + run: | + curl -s https://raw.githubusercontent.com/pgrange/bash_unit/master/install.sh | bash + FORCE_COLOR=true ./bash_unit tests/test_* diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml new file mode 100644 index 0000000..e15a20c --- /dev/null +++ b/.github/workflows/linting.yml @@ -0,0 +1,41 @@ +name: Linting + +# Always run on Pull Requests as then these checks can be marked as required. +on: + push: + branches: + - master + pull_request: + +permissions: {} + +jobs: + shell-check: + name: ShellCheck + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Add error parser + run: echo -n "::add-matcher::shellcheck-matcher.json" + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@2.0.0 + with: + format: gcc + + linting-summary: + name: Linting Summary + runs-on: ubuntu-22.04 + needs: [shell-check] + if: (!cancelled()) + steps: + - name: Check if any job failed + run: | + if [[ -z "$(echo "${{ join(needs.*.result, '') }}" | sed -e 's/success//g')" ]]; then + echo "All jobs succeeded" + else + echo "One or more jobs did not succeed" + exit 1 + fi diff --git a/.github/workflows/rpm-build.yml b/.github/workflows/rpm-build.yml new file mode 100644 index 0000000..8bfda1e --- /dev/null +++ b/.github/workflows/rpm-build.yml @@ -0,0 +1,435 @@ +name: RPM Build + +on: + workflow_call: + inputs: + NAME: + type: string + required: true + DISTROS: + type: string + required: true + EL8_BUILD_VERSION: + type: string + required: true + EL9_BUILD_VERSION: + type: string + required: true + LEAP15_VERSION: + type: string + required: true + COVFN_DISABLED: + type: boolean + default: true + PACKAGING_DIR: + type: string + required: true + # allow the caller to decide to run in GHA or not + RUN_GHA: + type: boolean + default: false + # these are mainly used for self-testing + BRANCH: + type: string + description: "The branch of the project to build (normally empty to build the current project)" + required: false + UPDATE_PACKAGING: + type: boolean + description: "Whether to update the packaging/ dir (primarily used for self-testing)" + default: false + required: false + outputs: + rpm-test-version: + value: ${{ jobs.Import-commit-pragmas.outputs.rpm-test-version }} + pr-repos: + value: ${{ jobs.Import-commit-pragmas.outputs.pr-repos }} + run-gha: + value: ${{ jobs.Import-commit-pragmas.outputs.run-gha }} + skip-build: + value: ${{ jobs.Import-commit-pragmas.outputs.skip-build }} + commit-message: + value: ${{ jobs.Import-commit-message.outputs.message }} + dequoted-commit-message: + value: ${{ jobs.Import-commit-message.outputs.dequoted-message }} + +jobs: + # it's a real shame that this step is even needed. push events have the commit message in + # ${{ github.event.head_commit.message }} but pull_requests don't. :-( + Import-commit-message: + name: Get commit message + runs-on: [self-hosted, light] + # Map a step output to a job output + outputs: + message: ${{ steps.commit_message.outputs.text }} + dequoted-message: ${{ steps.dequoted_commit_message.outputs.text }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Import Commit Message + id: commit_message + run: exec >> $GITHUB_OUTPUT; + echo 'text<> $GITHUB_OUTPUT + # it's tempting to use the previous step's output as such: + # echo '$ {{ steps.commit_message.outputs.text }}' | escape_single_quotes; + # below, instead of using git again, but if that commit message has a ' in it, + # (which we are trying to escape here) it will cause a sh parsing error here + echo 'text<> $GITHUB_OUTPUT; \ + cat $GITHUB_OUTPUT' EXIT + if ${CP_SKIP_BUILD:-false}; then + exit 0 + fi + distros=(${{ inputs.DISTROS }}) + if ! ${CP_SKIP_BUILD_EL8_RPM:-false} && [[ ${distros[@]} =~ el8 ]]; then + l+=(el8) + fi + if ! ${CP_SKIP_BUILD_EL9_RPM:-false} && [[ ${distros[@]} =~ el9 ]]; then + l+=(el9) + fi + if ! ${CP_SKIP_BUILD_LEAP15_RPM:-false} && [[ ${distros[@]} =~ leap15 ]]; then + l+=(leap15) + fi + - name: Calculate RPM Build Matrix + id: matrix + # TODO: make this consume the previous step's output + run: | # do not use the non-| format for this script + l=() + trap 'echo "text=[$(IFS=","; echo "${l[*]}")]" >> $GITHUB_OUTPUT; \ + cat $GITHUB_OUTPUT' EXIT + if ${CP_SKIP_BUILD:-false}; then + exit 0 + fi + distros=(${{ inputs.DISTROS }}) + if ! ${CP_SKIP_BUILD_EL8_RPM:-false} && [[ ${distros[@]} =~ el8 ]]; then + l+=('"el8"') + fi + if ! ${CP_SKIP_BUILD_EL9_RPM:-false} && [[ ${distros[@]} =~ el9 ]]; then + l+=('"el9"') + fi + if ! ${CP_SKIP_BUILD_LEAP15_RPM:-false} && [[ ${distros[@]} =~ leap15 ]]; then + l+=('"leap15"') + fi + + Build-RPM: + name: Build RPM + permissions: + statuses: write + runs-on: [self-hosted, docker] + needs: [Create-symlinks, Import-commit-pragmas, Calc-rpm-build-matrix, + Import-commit-message] + strategy: + matrix: + distro: ${{ fromJSON(needs.Calc-rpm-build-matrix.outputs.matrix) }} + fail-fast: false + if: | + needs.Create-symlinks.result == 'success' && + ((!cancelled()) || success() || failure()) + env: + ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ + DAOS_EMAIL: brian.murrell@intel.com + DAOS_FULLNAME: daos-stack + DISTRO: ${{ matrix.distro }} + DISTRO_REPOS: disabled + DOCKER_BUILDKIT: 0 + JENKINS_URL: https://build.hpdd.intel.com/ + ARTIFACTS_URL: file:///scratch/job_repos/ + MOCK_OPTIONS: --uniqueext=${{ github.run_id }} + PR_NUM: ${{ github.event.pull_request.number }} + # TODO -- this should be on stable, backedup storage, not /scratch + # yamllint disable-line rule:line-length + REPO_PATH: /scratch/job_repos/daos-stack/job/${{ inputs.NAME }}/job/PR-${{ github.event.pull_request.number }}/ + REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ + RUN_ID: ${{ github.run_id }} + TARGET: ${{ matrix.distro }} + # keep VS Code's GHA linting happy + STAGE_NAME: + DISTRO_NAME: + DISTRO_VERSION: + CP_LEAP15_VERSION: + COMMIT_STATUS_DISTRO_VERSION: + FVERSION: + CHROOT_NAME: + ACT: + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + repository: daos-stack/${{ inputs.NAME }} + ref: ${{ inputs.BRANCH && inputs.BRANCH || github.event.pull_request.head.sha }} + - name: Update packaging/ if requested + if: inputs.UPDATE_PACKAGING + uses: actions/checkout@v4 + with: + repository: daos-stack/packaging + path: packaging + - name: Import commit pragmas + uses: daos-stack/action-import-commit-pragmas@v1 + with: + commit-message: ${{ needs.Import-commit-message.outputs.dequoted-message }} + - name: Set variables + run: | + FVERSION="37" + case ${{ matrix.distro }} in + 'el8') + CHROOT_NAME="rocky+epel-8-x86_64" + DISTRO_NAME="EL" + DISTRO_VERSION='${{ inputs.EL8_BUILD_VERSION }}' + COMMIT_STATUS_DISTRO_VERSION="8" + ;; + 'el9') + CHROOT_NAME="rocky+epel-9-x86_64" + DISTRO_NAME="EL" + DISTRO_VERSION='${{ inputs.EL9_BUILD_VERSION }}' + ;; + 'leap15') + CHROOT_NAME='opensuse-leap-${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || + inputs.LEAP15_VERSION }}-x86_64' + DISTRO_NAME="Leap" + DISTRO_VERSION='${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || inputs.LEAP15_VERSION }}' + ;; + esac + echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV + echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV + echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV + echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-"'${{ github.run_id }}/' >> $GITHUB_ENV + echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV + echo "FVERSION=$FVERSION" >> $GITHUB_ENV + echo "COMMIT_STATUS_DISTRO_VERSION=$COMMIT_STATUS_DISTRO_VERSION" >> $GITHUB_ENV + - name: Build RPM Docker image + id: build-rpm-docker-image + continue-on-error: true + run: | + set -eux + if "${ACT:-false}" && command -v podman; then + DOCKER=podman + else + DOCKER=docker + fi + "$DOCKER" build --file ${{ inputs.PACKAGING_DIR }}/packaging/Dockerfile.mockbuild \ + --build-arg CACHEBUST=$(date +%s%3N) \ + --build-arg CB0=$(date +%V) \ + --build-arg REPO_FILE_URL=$REPO_FILE_URL \ + --build-arg UID=$(id -u) \ + --build-arg FVERSION=${{ env.FVERSION }} \ + --tag mock-build \ + ${{ inputs.PACKAGING_DIR }} + - name: Build RPM + id: build-rpm + continue-on-error: true + # yamllint disable rule:line-length + run: | + set -eux + rm -rf mock_result/${CHROOT_NAME} + mkdir -p mock_result/${CHROOT_NAME} + if [ -e ci/rpm/build.sh ]; then + script="ci/rpm/build.sh" + else + script="set -eux; echo \"In docker I am \$(id)\" + if ! rm -rf artifacts/$TARGET/ || + ! mkdir -p artifacts/$TARGET/; then + echo \"Failed to create artifacts/$TARGET/\" + exit 1 + fi + make CHROOT_NAME=\"$CHROOT_NAME\" \ + DISTRO_VERSION=\"$DISTRO_VERSION\" chrootbuild" + fi + if [ -z "${{ github.run_attempt }}" ] || + [ "${{ github.run_attempt }}" = '""' ]; then + # probably running in act + unique="$RANDOM" + else + unique="${{ github.run_id }}-${{ github.run_attempt }}" + fi + if "${ACT:-false}" && command -v podman; then + DOCKER=podman + else + DOCKER=docker + # TODO: determine why we cannot use this with podman + docker_args=(--user build) + fi + "$DOCKER" run --name mock-build-"$unique"-${{ matrix.distro }} \ + "${docker_args[@]}" \ + --rm \ + -v "$PWD":"$PWD" -w "$PWD" \ + -v "$PWD"/mock_result/${CHROOT_NAME}:/var/lib/mock/$CHROOT_NAME/result \ + -v /scratch:/scratch \ + --privileged=true \ + -e DAOS_FULLNAME="$DAOS_FULLNAME" \ + -e DAOS_EMAIL="$DAOS_EMAIL" \ + -e DISTRO_VERSION="$DISTRO_VERSION" \ + -e STAGE_NAME="$STAGE_NAME" \ + -e CHROOT_NAME="$CHROOT_NAME" \ + -e ARTIFACTORY_URL="$ARTIFACTORY_URL" \ + -e REPO_FILE_URL="$REPO_FILE_URL" \ + -e JENKINS_URL="$JENKINS_URL" \ + -e TARGET="$TARGET" \ + -e COVFN_DISABLED="${{ inputs.COVFN_DISABLED }}" \ + mock-build bash -c "$script" + # yamllint enable rule:line-length + - name: Build RPM failure log + id: build-rpm-fail-log + continue-on-error: true + if: steps.build-rpm.outcome != 'success' + run: cat mock_result/${CHROOT_NAME}/root.log; + cat mock_result/${CHROOT_NAME}/build.log + - name: Save RPM build logs + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} logs + path: | + mock_result/${{ env.CHROOT_NAME }}/root.log + mock_result/${{ env.CHROOT_NAME }}/build.log + - name: Create repo + id: create-repo + if: steps.build-rpm.outcome == 'success' + continue-on-error: true + run: | + set -eux + repo_path="$REPO_PATH$GITHUB_RUN_NUMBER/artifact/artifacts/${{ !inputs.COVFN_DISABLED && 'bullseye/' || '' }}" + mkdir -p "$repo_path$TARGET" + cp -a mock_result/${CHROOT_NAME}/*.rpm "$repo_path$TARGET" + cd "$repo_path$TARGET" + createrepo . + - name: Test repo + id: test-repo + if: steps.create-repo.outcome == 'success' + continue-on-error: true + run: | + dnf --disablerepo=\* --repofrompath \ + testrepo,file://${REPO_PATH}${{ github.run_number }}/artifact/artifacts/${{ !inputs.COVFN_DISABLED && 'bullseye/' || '' }}$TARGET \ + repoquery -a + - name: Remove lastSuccessfulBuild link and exit failure + if: steps.test-repo.outcome != 'success' + run: rm -f ${REPO_PATH}lastSuccessfulBuild; + exit 1 + # We can't actually do this due to SDL requirements + # - name: Publish RPMs + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ env.DISTRO_NAME }} ${{ env.DISTRO_VERSION }} RPM repository + # path: ${{ env.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ env.TARGET }} + - name: Update commit status + if: contains(fromJson('["push", "pull_request"]'), github.event_name) && !env.ACT + uses: ouzi-dev/commit-status-updater@v2 + with: + # yamllint disable-line rule:line-length + name: 'build/Build RPM on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' + status: "${{ job.status }}" diff --git a/.github/workflows/rpm-test.yml b/.github/workflows/rpm-test.yml new file mode 100644 index 0000000..7bd86ea --- /dev/null +++ b/.github/workflows/rpm-test.yml @@ -0,0 +1,74 @@ +name: RPM Test + +on: + workflow_call: + inputs: + NAME: + type: string + required: true + DISTROS: + type: string + required: true + TEST_TAG: + type: string + required: true + # allow the caller to decide to run in GHA or not + RUN_GHA: + type: boolean + default: false + +permissions: {} + +jobs: + Test-with-DAOS: + # TODO: investigate how cancelling this can cancel the downstream job + name: Test RPMs with DAOS + runs-on: [self-hosted, mockbuilder] + if: | + (inputs.RUN_GHA || + github.event_name == 'workflow_dispatch') && + ((!cancelled()) || success() || failure()) + permissions: + issues: write + timeout-minutes: 7200 + strategy: + fail-fast: false + matrix: + # no GHA support on this branch: branch: [master, release/2.4] + # DO NOT LAND - using PR as master for now + # branch: [master] + branch: [bmurrell/run-on-dispatch] + env: + PR_REPO: >- + ${{ inputs.NAME }}@PR-${{ github.event.pull_request.number }}:${{ github.run_number }} + steps: + - name: Test RPMs with DAOS + uses: convictional/trigger-workflow-and-wait@v1.6.5 + with: + owner: daos-stack + repo: daos + github_token: ${{ secrets.GHA_WORKFLOW_TRIGGER }} + comment_downstream_url: ${{ github.event.pull_request.comments_url }} + workflow_file_name: 'rpm-build-and-test.yml' + # TODO: or rather I suspect it's a TODO, but I suspect we need to create + # a temporary branch here so as not to pollute the landing branch + # runs with these tests + ref: ${{ matrix.branch }} + wait_interval: 10 + # TODO: rpm-test-version of course needs to either be the latest + # version, or better yet, no version and it installs the latest + # version + client_payload: '{"pr-repos": + "${{ env.PR_REPO }}", + "commit-message": + "Override commit pragmas", + "test-tag": + "load_mpi test_core_files ${{ inputs.TEST_TAG }}", + "functional-test-distros": + "${{ inputs.DISTROS }}" + }' + propagate_failure: true + # TODO: investigate how cancelling this can cancel the downstream job + - name: Cleanup downstream jobs + if: cancelled() + run: echo "Cancelling downstream jobs (but how?)" diff --git a/.github/workflows/test-rpm-build.yml b/.github/workflows/test-rpm-build.yml new file mode 100644 index 0000000..8cec8ac --- /dev/null +++ b/.github/workflows/test-rpm-build.yml @@ -0,0 +1,70 @@ +name: Test RPM Build Reusable Workflow + +env: + # TODO: we really need to define a list of supported versions (ideally it's no more than 2) + # build is done on the lowest version and test on the highest with a "sanity test" + # stage done on all versions in the list ecept the highest + EL8_BUILD_VERSION: 8.6 + EL8_VERSION: 8.8 + EL9_BUILD_VERSION: 9 + EL9_VERSION: 9 + LEAP15_VERSION: 15.5 + # Which distros to build for + DISTROS: el8 el9 leap15 + PACKAGING_DIR: . + +on: + push: + branches: + - master + pull_request: + paths: + - .github/workflows/rpm-build.yml + - .github/workflows/test-rpm-build.yml + +concurrency: + group: test-rpm-build-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash --noprofile --norc -ueo pipefail {0} + +permissions: {} + +jobs: + Variables: + # What a dumb jobs this is + # Needed because of https://github.com/orgs/community/discussions/26671 + # Ideally want to be able to use: + # with: + # NAME: ${{ env.DISTROS }} + # in the Call-RPM-Build job but the above issue prevents it + name: Compute outputs + runs-on: [self-hosted, light] + outputs: + DISTROS: ${{ env.DISTROS }} + EL8_BUILD_VERSION: ${{ env.EL8_BUILD_VERSION }} + EL9_BUILD_VERSION: ${{ env.EL9_BUILD_VERSION }} + LEAP15_VERSION: ${{ env.LEAP15_VERSION }} + PACKAGING_DIR: ${{ env.PACKAGING_DIR }} + steps: + - name: Make outputs from env variables + run: echo "Make outputs from env variables" + + Call-RPM-Build: + name: Build RPM + needs: Variables + permissions: + statuses: write + uses: ./.github/workflows/rpm-build.yml + secrets: inherit + with: + NAME: argobots + DISTROS: ${{ needs.Variables.outputs.DISTROS }} + EL8_BUILD_VERSION: ${{ needs.Variables.outputs.EL8_BUILD_VERSION }} + EL9_BUILD_VERSION: ${{ needs.Variables.outputs.EL9_BUILD_VERSION }} + LEAP15_VERSION: ${{ needs.Variables.outputs.LEAP15_VERSION }} + PACKAGING_DIR: ${{ needs.Variables.outputs.PACKAGING_DIR }} + UPDATE_PACKAGING: true + BRANCH: master diff --git a/get_commit_pragmas b/get_commit_pragmas new file mode 100755 index 0000000..c6bb477 --- /dev/null +++ b/get_commit_pragmas @@ -0,0 +1,7 @@ +#!/bin/bash + +sed -Ene 's/^([-[:alnum:]]+): *([-:<@>,\._ [:alnum:]]+)$/\1 \2/p' | while read -r a b; do + echo -n "${a//-/_}" | tr '[:lower:]' '[:upper:]' + # escape special characters in the value + echo "=$b" | sed -e 's/\([<> ]\)/\\\1/g' +done diff --git a/shellcheck-matcher.json b/shellcheck-matcher.json new file mode 100644 index 0000000..1b8b487 --- /dev/null +++ b/shellcheck-matcher.json @@ -0,0 +1,17 @@ +{ + "problemMatcher": [ + { + "owner": "shellcheck-problem-matcher", + "severity": "error", + "pattern": [ + { + "regexp": "^(.*?):(\\d+):(\\d*):?\\s+(?:fatal\\s+)?(warning|error|note):\\s+(.*)$", + "file": 1, + "line": 2, + "column": 3, + "message": 5 + } + ] + } + ] +} diff --git a/tests/test_get_commit_pragmas b/tests/test_get_commit_pragmas new file mode 100755 index 0000000..8edc692 --- /dev/null +++ b/tests/test_get_commit_pragmas @@ -0,0 +1,71 @@ +#!/bin/bash + +set -eu + +test_get_commit_pragmas() { + local msg='Use external action + +Run-GHA: true +Skip-PR-comments: true +Test-tag: always_passes,vm + +Required-githooks: true + +Signed-off-by: Brian J. Murrell +' + assert_equals "$(echo "$msg" | ../get_commit_pragmas)" 'RUN_GHA=true +SKIP_PR_COMMENTS=true +TEST_TAG=always_passes,vm +REQUIRED_GITHOOKS=true +SIGNED_OFF_BY=Brian\ J.\ Murrell\ \' + + local msg='Escape spaces also + +'"'"'Will-not-be-a-pragma: false'"'"' should not be considered a commit +pragma, but: +Should-not-be-a-pragma: bar will be because it was not quoted. + +Skip-func-test-leap15: false +RPM-test-version: 2.5.100-13.10036.g65926e32 +Skip-PR-comments: true +Test-tag: always_passes always_fails +EL8-VM9-label: all_vm9 +EL9-VM9-label: all_vm9 +Leap15-VM9-label: all_vm9 +HW-medium-label: new_icx5 +HW-large-label: new_icx9 + +Required-githooks: true + +Signed-off-by: Brian J. Murrell +' + assert_equals "$(echo "$msg" | ../get_commit_pragmas)" 'SHOULD_NOT_BE_A_PRAGMA=bar\ will\ be\ because\ it\ was\ not\ quoted. +SKIP_FUNC_TEST_LEAP15=false +RPM_TEST_VERSION=2.5.100-13.10036.g65926e32 +SKIP_PR_COMMENTS=true +TEST_TAG=always_passes\ always_fails +EL8_VM9_LABEL=all_vm9 +EL9_VM9_LABEL=all_vm9 +LEAP15_VM9_LABEL=all_vm9 +HW_MEDIUM_LABEL=new_icx5 +HW_LARGE_LABEL=new_icx9 +REQUIRED_GITHOOKS=true +SIGNED_OFF_BY=Brian\ J.\ Murrell\ \' + + local msg='Test PR-repos pragma + +PR-repos: argobots@PR-23:91 + +Required-githooks: true + +Signed-off-by: Brian J. Murrell +Run-GHA: true +Skip-PR-comments: true +' + assert_equals "$(echo "$msg" | ../get_commit_pragmas)" 'PR_REPOS=argobots@PR-23:91 +REQUIRED_GITHOOKS=true +SIGNED_OFF_BY=Brian\ J.\ Murrell\ \ +RUN_GHA=true +SKIP_PR_COMMENTS=true' + +}