Merge branch 'master' into aaoganez/DAOS-15595

.dockerignore

@@ -24,7 +24,9 @@
 !ftest.sh
 !site_scons
 !.clang-format
-!requirements.txt
+!requirements-build.txt
+!requirements-utest.txt
+!requirements-ftest.txt
 
 # Now disallow extra files which may be present but are not required.
 # Include the rdb repo as well.

.github/actions/provision-cluster/action.yml

@@ -40,7 +40,8 @@ runs:
       run: |
         . ci/gha_functions.sh
         inst_repos="${{ env.CP_PR_REPOS }} ${{ github.event.inputs.pr-repos }}"
-        if [[ $inst_repos != *daos@* ]]; then
+        if [ -z "${{ env.CP_RPM_TEST_VERSION }}" ] &&
+           [[ $inst_repos != *daos@* ]]; then
             inst_repos+=" daos@PR-${{ github.event.pull_request.number }}"
             inst_repos+=":${{ github.run_number }}"
         fi

.github/workflows/bash_unit_testing.yml

@@ -11,6 +11,8 @@ defaults:
   run:
     shell: bash --noprofile --norc -ueo pipefail {0}
 
+permissions: {}
+
 jobs:
   Test-gha-functions:
     name: Tests in ci/gha_functions.sh

.github/workflows/ci2.yml

@@ -7,13 +7,19 @@ concurrency:
   group: ci2-${{ github.head_ref }}
   cancel-in-progress: true
 
+permissions: {}
+
 jobs:
 
   # reuse the cache from the landing-builds workflow if available, if not then build the images
   # from scratch, but do not save them.
   Build-and-test:
     name: Run DAOS/NLT tests
     runs-on: ubuntu-22.04
+    permissions:
+      # https://github.com/EnricoMi/publish-unit-test-result-action#permissions
+      checks: write
+      pull-requests: write
     strategy:
       matrix:
         distro: [ubuntu]

.github/workflows/clang-format.yml

@@ -18,6 +18,8 @@ name: clang-format
 on:
   pull_request:
 
+permissions: {}
+
 jobs:
   pylint:
     name: Clang Format

.github/workflows/create_release.yml

@@ -8,6 +8,8 @@ on:
       - master
       - 'release/**'
 
+permissions: {}
+
 jobs:
   make_release:
     name: Create Release

.github/workflows/doxygen.yml

@@ -4,6 +4,8 @@ name: Doxygen
 on:
   pull_request:
 
+permissions: {}
+
 jobs:
 
   Doxygen:

.github/workflows/flake.yml

@@ -4,6 +4,8 @@ name: Flake
 on:
   pull_request:
 
+permissions: {}
+
 jobs:
   flake8-lint:
     runs-on: ubuntu-22.04

.github/workflows/landing-builds.yml

@@ -14,7 +14,10 @@ on:
       - 'utils/scripts/helpers/*'
       - utils/ci
       - ci
-      - requirements.txt
+      - requirements-build.txt
+      - requirements-utest.txt
+
+permissions: {}
 
 jobs:
 
@@ -85,6 +88,10 @@ jobs:
     name: Run DAOS/NLT tests
     needs: Prepare
     runs-on: ubuntu-22.04
+    permissions:
+      # https://github.com/EnricoMi/publish-unit-test-result-action#permissions
+      checks: write
+      pull-requests: write
     strategy:
       matrix:
         distro: [ubuntu]

.github/workflows/linting.yml

@@ -9,6 +9,8 @@ on:
       - 'release/*'
   pull_request:
 
+permissions: {}
+
 jobs:
   # Run isort on the tree.
   # This checks .py files only so misses SConstruct and SConscript files are not checked, rather
@@ -26,7 +28,7 @@ jobs:
           python-version: '3'
       - uses: isort/isort-action@master
         with:
-          requirementsFiles: "requirements.txt utils/cq/requirements.txt"
+          requirementsFiles: "requirements.txt"
       - name: Run on SConstruct file.
         run: isort --check-only SConstruct
       - name: Run on build files.

.github/workflows/ossf-scorecard.yml

@@ -0,0 +1,76 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: Scorecard supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: '45 8 * * 0'
+  push:
+    branches: ["master"]
+  pull_request:
+
+# Declare default permissions as nothing.
+permissions: {}
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+      # Uncomment the permissions below if installing in a private repository.
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v4.1.1
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736  # v2.3.1
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecard on a *private* repository
+          # To create the PAT, follow the steps in
+          # https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional.
+          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+          # Public repositories:
+          #   - Publish results to OpenSSF REST API for easy access by consumers
+          #   - Allows the repository to include the Scorecard badge.
+          #   - See https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories:
+          #   - `publish_results` will always be set to `false`, regardless
+          #     of the value entered here.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable
+      # uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@97a0fba1372883ab732affbe8f94b823f91727db  # v3.pre.node20
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard (optional).
+      # Commenting out will disable upload of results to your repo's Code Scanning dashboard
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@1b1aada464948af03b950897e5eb522f92603cc2  # v3.24.9
+        with:
+          sarif_file: results.sarif

.github/workflows/pr-metadata.yml

@@ -9,10 +9,13 @@ on:
   pull_request_target:
     types: [opened, synchronize, reopened, edited]
 
+permissions: {}
 
 jobs:
   example_comment_pr:
     runs-on: ubuntu-22.04
+    permissions:
+      pull-requests: write
     name: Report Jira data to PR comment
     steps:
       - name: Checkout

.github/workflows/pylint.yml

@@ -5,6 +5,8 @@ name: Pylint
 on:
   pull_request:
 
+permissions: {}
+
 jobs:
   pylint:
     name: Pylint check

.github/workflows/rpm-build-and-test-report.yml

@@ -8,14 +8,14 @@ on:
   # for testing before landing
   workflow_dispatch:
 
-permissions:
-  contents: read
-  actions: read
-  checks: write
+permissions: {}
 
 jobs:
   report-vm-1:
     runs-on: [self-hosted, docker]
+    # https://github.com/dorny/test-reporter/issues/149
+    permissions:
+      checks: write
     strategy:
       matrix:
         # TODO: figure out how to determine this matrix

.github/workflows/rpm-build-and-test.yml

@@ -26,13 +26,7 @@ defaults:
   run:
     shell: bash --noprofile --norc -ueo pipefail {0}
 
-# https://github.com/dorny/test-reporter/issues/149
-permissions:
-  id-token: write
-  contents: read
-  checks: write
-  # https://github.com/EnricoMi/publish-unit-test-result-action#permissions
-  pull-requests: write
+permissions: {}
 
 jobs:
   # it's a real shame that this step is even needed.  push events have the commit message # in
@@ -363,6 +357,10 @@ jobs:
   Functional:
     name: Functional Testing
     runs-on: [self-hosted, wolf]
+    permissions:
+      # https://github.com/EnricoMi/publish-unit-test-result-action#permissions
+      checks: write
+      pull-requests: write
     timeout-minutes: 7200
     needs: [Build-RPM, Import-commit-message, Calc-functional-matrix, Import-commit-pragmas]
     strategy:
@@ -594,6 +592,10 @@ jobs:
   Functional_Hardware:
     name: Functional Testing on Hardware
     runs-on: [self-hosted, wolf]
+    permissions:
+      # https://github.com/EnricoMi/publish-unit-test-result-action#permissions
+      checks: write
+      pull-requests: write
     timeout-minutes: 7200
     needs: [Import-commit-message, Build-RPM, Calc-functional-hardware-matrix,
             Import-commit-pragmas, Functional]

.github/workflows/spelling.yml

@@ -3,6 +3,8 @@ name: Codespell
 on:
   pull_request:
 
+permissions: {}
+
 jobs:
 
   Codespell:

.github/workflows/version-checks.yml

@@ -8,6 +8,8 @@ on:
     paths:
       - 'utils/cq/requirements.txt'
 
+permissions: {}
+
 jobs:
   upgrade-check:
     name: Check for updates
@@ -19,6 +21,9 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
       - name: Install extra python packages
         run: python3 -m pip install --requirement utils/cq/requirements.txt
       - name: Check ${{ matrix.package }} version

.github/workflows/yaml.yml

@@ -13,6 +13,8 @@ on:
       - '**/*.yml'
       - utils/cq/requirements.txt
 
+permissions: {}
+
 jobs:
   yaml-lint:
     runs-on: ubuntu-22.04

Jenkinsfile

@@ -108,7 +108,8 @@ void fixup_rpmlintrc() {
                     '/usr/bin/hello_drpc',
                     '/usr/bin/daos_firmware',
                     '/usr/bin/daos_admin',
-                    '/usr/bin/daos_server']
+                    '/usr/bin/daos_server',
+                    '/usr/bin/ddb']
 
     String content = readFile(file: 'utils/rpms/daos.rpmlintrc') + '\n\n' +
                      '# https://daosio.atlassian.net/browse/DAOS-11534\n'
@@ -1016,7 +1017,8 @@ pipeline {
                     post {
                         always {
                             discoverGitReferenceBuild referenceJob: 'daos-stack/daos/master',
-                                                      scm: 'daos-stack/daos'
+                                                      scm: 'daos-stack/daos',
+                                                      requiredResult: hudson.model.Result.UNSTABLE
                             recordIssues enabledForFailure: true,
                                          failOnError: false,
                                          ignoreQualityGate: true,

SConstruct

@@ -480,9 +480,8 @@ def scons():
     prereqs.save_build_info()
     # also install to $PREFIX/lib to work with existing avocado test code
     if prereqs.test_requested():
-        env.Install('$PREFIX/lib/daos', ['.build_vars.sh', '.build_vars.json'])
         env.Install('$PREFIX/lib/daos/TESTING/ftest/util', ['site_scons/env_modules.py'])
-        env.Install('$PREFIX/lib/daos/TESTING/ftest/', ['ftest.sh'])
+        env.Install('$PREFIX/lib/daos/TESTING/ftest/', ['ftest.sh', "requirements-ftest.txt"])
 
     env.Install("$PREFIX/lib64/daos", "VERSION")
 

ci/codespell.ignores

@@ -34,3 +34,5 @@ expres
 signalling
 laf
 cacl
+chk
+falloc

ci/gha_helper.py

@@ -16,6 +16,8 @@
                '.github/workflows/landing-builds.yml',
                '.dockerignore',
                'requirements.txt',
+               'requirements-build.txt',
+               'requirements-utest.txt',
                'ci/gha_helper.py']
 
 COMMIT_CMD = ['git', 'rev-parse', '--short', 'HEAD']