Merge pull request #6085 from osalyk/release_2.1.0

Release 2.1.0

.github/actions/pmem_benchmark_run/action.yml

@@ -0,0 +1,45 @@
+name: PMEM benchmark procedure
+description: PMDK benchmarking procedure for self-hosted runners equipped with PMEM
+inputs:
+  runtime_dir:
+    description: The root directory of the repository designated as runtime
+    required: true
+  reference_LIB_PATH:
+    description: LD_LIBRARY_PATH where the first version of PMDK is built
+    required: true
+  rival_LIB_PATH:
+    description: LD_LIBRARY_PATH where the second version of PMDK is built
+    required: true
+  config:
+    description: Name of the .cfg file to use
+    required: true
+  scenario:
+    description: Name of the scenario to run
+    required: true
+  pmem_path:
+    description: A PMEM-mounted directory to use
+    default: /mnt/pmem0
+runs:
+  using: composite
+  steps:
+    - name: Run the benchmark
+      working-directory: ${{ inputs.runtime_dir }}
+      shell: bash
+      run: >-
+        ./utils/benchmarks/run_and_combine.py
+        --reference ${{ inputs.reference_LIB_PATH }} --rival ${{ inputs.rival_LIB_PATH }}
+        --config ${{ inputs.config }} --scenario ${{ inputs.scenario }}
+        --pmem_path ${{ inputs.pmem_path }}
+
+    - name: Archive logs
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ inputs.config }}__${{ inputs.scenario }}
+        path: '${{ inputs.runtime_dir }}/*.csv'
+
+    - name: Remove logs
+      if: always()
+      working-directory: ${{ inputs.runtime_dir }}
+      shell: bash
+      run: rm -f *.csv

.github/actions/pmem_test_prepare/action.yml

@@ -9,6 +9,10 @@ inputs:
     description: Build with the fault injection capability
     required: false
     default: '0'
+  valgrind:
+    description: Build with Valgrind support
+    required: true
+
 runs:
   using: composite
   steps:
@@ -30,6 +34,9 @@ runs:
     - env:
         FAULT_INJECTION: ${{ inputs.fault_injection }}
         NDCTL_ENABLE: ${{ inputs.ndctl_enable }}
+        PMEMOBJ_IGNORE_DIRTY_SHUTDOWN: ${{ inputs.ndctl_enable == 'n' && 'y' || 'n' }}
+        PMEMOBJ_IGNORE_BAD_BLOCKS: ${{ inputs.ndctl_enable == 'n' && 'y' || 'n' }}
+        VALGRIND: ${{ inputs.valgrind }}
       run: |
         echo '::group::Build'
         $WORKDIR/build-pmdk.sh

.github/workflows/docker_rebuild.yml

@@ -37,7 +37,7 @@ jobs:
           - {OS: ubuntu, OS_VER: 22.04}
     steps:
       - name: Clone the git repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Rebuild the image
         env:

.github/workflows/main.yml

@@ -13,12 +13,16 @@ jobs:
       issues: read
     steps:
       - name: Clone the git repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           # required for `make check-license` to work properly
           fetch-depth: 50
 
       - name: Check changelog
+        # Skip for pmem/pmdk/master and stable-* branches
+        if: |
+          !(github.repository == 'pmem/pmdk' &&
+            (github.ref_name == 'master' || startsWith(github.ref_name, 'stable-')))
         uses: Zomzog/changelog-checker@v1.3.0
         with:
           fileName: ChangeLog
@@ -32,31 +36,65 @@ jobs:
           sudo pip install flake8
           sudo apt-get install clang-format-14
 
-      - name: Check licenses
+      - name: Check license
+        id: check_license
+        continue-on-error: true
         env:
           NDCTL_ENABLE: n # just to speed up the job
-        run: make -j$(nproc) check-license
+          PMEMOBJ_IGNORE_DIRTY_SHUTDOWN: y # not recommended for production
+          PMEMOBJ_IGNORE_BAD_BLOCKS: y # not recommended for production
+        run: |
+          make -j$(nproc) check-license || true
+          git diff > /tmp/check-license.diff
+          [ $(cat /tmp/check-license.diff | wc -l ) -gt 0 ] && exit 1
+          exit 0
+
+      - name: Upload check license diff
+        if: steps.check_license.outcome != 'success'
+        uses: actions/upload-artifact@v4
+        with:
+          name: check-license.diff
+          path: /tmp/check-license.diff
+
+      - name: Check license - Exit code
+        run: |
+          [ "${{steps.check_license.outcome}}" != "success" ] && exit 1
+          exit 0
 
       - name: Check style
         env:
           CSTYLE_FAIL_IF_CLANG_FORMAT_MISSING: 1
           NDCTL_ENABLE: n # just to speed up the job
+          PMEMOBJ_IGNORE_DIRTY_SHUTDOWN: y # not recommended for production
+          PMEMOBJ_IGNORE_BAD_BLOCKS: y # not recommended for production
         run: make -j$(nproc) cstyle
 
 
   basic_build:
     name: Basic build
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - CC: gcc
+            CXX: g++
+          - CC: clang
+            CXX: clang++
     steps:
       - name: Clone the git repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install dependencies
         run: sudo apt-get -y install pandoc
 
       - name: Build sources
         env:
           NDCTL_ENABLE: n # just to speed up the job
+          PMEMOBJ_IGNORE_DIRTY_SHUTDOWN: y # not recommended for production
+          PMEMOBJ_IGNORE_BAD_BLOCKS: y # not recommended for production
+          CC: ${{ matrix.CC }}
+          CXX: ${{ matrix.CXX }}
         run: make -j$(nproc) test
 
 
@@ -66,6 +104,8 @@ jobs:
     name: Ubuntu
 
   coverage_scan:
-    needs: call-ubuntu
+    needs: [src_checkers, basic_build]
     uses: ./.github/workflows/scan_coverage.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
     name: Coverage

.github/workflows/nightly.yml

@@ -35,7 +35,7 @@ jobs:
         TEST_BUILD: [debug, nondebug]
     steps:
       - name: Clone the git repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 50
 
@@ -66,7 +66,7 @@ jobs:
                 ]
     steps:
       - name: Clone the git repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 50
 

.github/workflows/pmem.io_doc_update.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Clone the git repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install dependencies
         run: sudo apt-get install libndctl-dev libdaxctl-dev pandoc

.github/workflows/pmem_benchmark.yml

@@ -0,0 +1,115 @@
+name: PMEM Benchmark
+
+on:
+  workflow_dispatch:
+    inputs:
+      reference_ref:
+        type: string
+        default: stable-2.0
+      rival_ref:
+        type: string
+        default: master
+
+
+jobs:
+  prep_runtime:
+    name: Prepare runtime
+    runs-on: [self-hosted, benchmark]
+    permissions:
+      contents: read
+    steps:
+      - name: Clone the git repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Build
+        run: make -j
+
+
+  prep_contesters:
+    name: Prepare ${{ matrix.ROLE }} (${{ matrix.GITHUB_REF }})
+    runs-on: [self-hosted, benchmark]
+    needs: prep_runtime
+    strategy:
+      matrix:
+        include:
+          - ROLE: reference
+            GITHUB_REF: ${{ inputs.reference_ref }}
+          - ROLE: rival
+            GITHUB_REF: ${{ inputs.rival_ref }}
+    permissions:
+      contents: read
+    env:
+      MANIFEST: ${{ matrix.ROLE }}/manifest.txt
+    steps:
+      - name: Clone the git repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ matrix.GITHUB_REF }}
+          fetch-depth: 1
+          path: ${{ matrix.ROLE }}
+
+      - name: Build
+        working-directory: ${{ matrix.ROLE }}
+        run: make -j
+
+      - name: Write the manifest
+        run: |
+          echo "${{ matrix.GITHUB_REF }}" >> $MANIFEST
+          git -C ${{ matrix.ROLE }} rev-parse HEAD >> $MANIFEST
+
+      - name: Archive the manifest
+        uses: actions/upload-artifact@v4
+        with:
+          name: manifest_${{ matrix.ROLE }}
+          path: ${{ env.MANIFEST }}
+
+
+  run:
+    name: Run perf.cfg ${{ matrix.SCENARIO }}
+    runs-on: [self-hosted, benchmark]
+    needs: prep_contesters
+    strategy:
+      matrix:
+        SCENARIO:
+          - obj_tx_alloc_small_v_thread
+          - obj_pmalloc_small_v_threads
+          - obj_rbtree_map_insert
+          - obj_hashmap_tx_map_insert
+    steps:
+      - name: Benchmark
+        uses: ./.github/actions/pmem_benchmark_run
+        with:
+          runtime_dir: ./
+          reference_LIB_PATH: reference/src/nondebug
+          rival_LIB_PATH: rival/src/nondebug
+          config: perf
+          scenario: ${{ matrix.SCENARIO }}
+
+
+  repack:
+    name: Repack
+    runs-on: ubuntu-latest
+    needs: run
+    steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: csvs
+
+      - name: Combine the manifests
+        env:
+          OUTPUT: csvs/manifest.txt
+        run: |
+          for competitor in reference rival; do
+            echo "$competitor:" >> $OUTPUT
+            cat csvs/manifest_$competitor/manifest.txt >> $OUTPUT
+            echo >> $OUTPUT
+          done
+
+      - name: Upload all as a single artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: perf__all__${{ github.run_id }}
+          path: csvs/**/*

.github/workflows/pmem_ras.yml

@@ -44,7 +44,7 @@ jobs:
 
     steps:
       - name: Clone the git repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Variables, such as $ras_runner are hidden on the controller platform as environmental variables.
       # 'sed' command is used to filter out IP addresses from the ansible output, it will show up as the 'ras_runner' instead.

.github/workflows/pmem_test_matrix.yml

@@ -9,6 +9,9 @@ on:
       force_enable:
         required: true
         type: string
+      valgrind:
+        required: true
+        type: string
       timeout_minutes:
         required: false
         type: number
@@ -28,11 +31,14 @@ jobs:
         os: [rhel, opensuse]
         build: [debug, nondebug]
 
+
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Test prepare
         uses: ./.github/actions/pmem_test_prepare
+        with:
+          valgrind: ${{ inputs.valgrind }}
 
       - name: Test run
         uses: ./.github/actions/pmem_test_run