Merge pull request #6085 from osalyk/release_2.1.0

Release 2.1.0

.github/actions/pmem_benchmark_run/action.yml

@@ -0,0 +1,45 @@
+name: PMEM benchmark procedure
+description: PMDK benchmarking procedure for self-hosted runners equipped with PMEM
+inputs:
+ runtime_dir:
+ description: The root directory of the repository designated as runtime
+ required: true
+ reference_LIB_PATH:
+ description: LD_LIBRARY_PATH where the first version of PMDK is built
+ required: true
+ rival_LIB_PATH:
+ description: LD_LIBRARY_PATH where the second version of PMDK is built
+ required: true
+ config:
+ description: Name of the .cfg file to use
+ required: true
+ scenario:
+ description: Name of the scenario to run
+ required: true
+ pmem_path:
+ description: A PMEM-mounted directory to use
+ default: /mnt/pmem0
+runs:
+ using: composite
+ steps:
+ - name: Run the benchmark
+ working-directory: ${{ inputs.runtime_dir }}
+ shell: bash
+ run: >-
+ ./utils/benchmarks/run_and_combine.py
+ --reference ${{ inputs.reference_LIB_PATH }} --rival ${{ inputs.rival_LIB_PATH }}
+ --config ${{ inputs.config }} --scenario ${{ inputs.scenario }}
+ --pmem_path ${{ inputs.pmem_path }}
+
+ - name: Archive logs
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ inputs.config }}__${{ inputs.scenario }}
+ path: '${{ inputs.runtime_dir }}/*.csv'
+
+ - name: Remove logs
+ if: always()
+ working-directory: ${{ inputs.runtime_dir }}
+ shell: bash
+ run: rm -f *.csv

.github/actions/pmem_test_prepare/action.yml

@@ -9,6 +9,10 @@ inputs:
  description: Build with the fault injection capability
  required: false
  default: '0'
+ valgrind:
+ description: Build with Valgrind support
+ required: true
+
 runs:
  using: composite
  steps:
@@ -30,6 +34,9 @@ runs:
  - env:
  FAULT_INJECTION: ${{ inputs.fault_injection }}
  NDCTL_ENABLE: ${{ inputs.ndctl_enable }}
+ PMEMOBJ_IGNORE_DIRTY_SHUTDOWN: ${{ inputs.ndctl_enable == 'n' && 'y' || 'n' }}
+ PMEMOBJ_IGNORE_BAD_BLOCKS: ${{ inputs.ndctl_enable == 'n' && 'y' || 'n' }}
+ VALGRIND: ${{ inputs.valgrind }}
  run: |
  echo '::group::Build'
  $WORKDIR/build-pmdk.sh

.github/workflows/docker_rebuild.yml

@@ -37,7 +37,7 @@ jobs:
  - {OS: ubuntu, OS_VER: 22.04}
  steps:
  - name: Clone the git repo
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
 
  - name: Rebuild the image
  env:

.github/workflows/main.yml

@@ -13,12 +13,16 @@ jobs:
  issues: read
  steps:
  - name: Clone the git repo
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  with:
  # required for `make check-license` to work properly
  fetch-depth: 50
 
  - name: Check changelog
+ # Skip for pmem/pmdk/master and stable-* branches
+ if: |
+ !(github.repository == 'pmem/pmdk' &&
+ (github.ref_name == 'master' || startsWith(github.ref_name, 'stable-')))
  uses: Zomzog/changelog-checker@v1.3.0
  with:
  fileName: ChangeLog
@@ -32,31 +36,65 @@ jobs:
  sudo pip install flake8
  sudo apt-get install clang-format-14
 
- - name: Check licenses
+ - name: Check license
+ id: check_license
+ continue-on-error: true
  env:
  NDCTL_ENABLE: n # just to speed up the job
- run: make -j$(nproc) check-license
+ PMEMOBJ_IGNORE_DIRTY_SHUTDOWN: y # not recommended for production
+ PMEMOBJ_IGNORE_BAD_BLOCKS: y # not recommended for production
+ run: |
+ make -j$(nproc) check-license || true
+ git diff > /tmp/check-license.diff
+ [ $(cat /tmp/check-license.diff | wc -l ) -gt 0 ] && exit 1
+ exit 0
+
+ - name: Upload check license diff
+ if: steps.check_license.outcome != 'success'
+ uses: actions/upload-artifact@v4
+ with:
+ name: check-license.diff
+ path: /tmp/check-license.diff
+
+ - name: Check license - Exit code
+ run: |
+ [ "${{steps.check_license.outcome}}" != "success" ] && exit 1
+ exit 0
 
  - name: Check style
  env:
  CSTYLE_FAIL_IF_CLANG_FORMAT_MISSING: 1
  NDCTL_ENABLE: n # just to speed up the job
+ PMEMOBJ_IGNORE_DIRTY_SHUTDOWN: y # not recommended for production
+ PMEMOBJ_IGNORE_BAD_BLOCKS: y # not recommended for production
  run: make -j$(nproc) cstyle
 
 
  basic_build:
  name: Basic build
  runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - CC: gcc
+ CXX: g++
+ - CC: clang
+ CXX: clang++
  steps:
  - name: Clone the git repo
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
 
  - name: Install dependencies
  run: sudo apt-get -y install pandoc
 
  - name: Build sources
  env:
  NDCTL_ENABLE: n # just to speed up the job
+ PMEMOBJ_IGNORE_DIRTY_SHUTDOWN: y # not recommended for production
+ PMEMOBJ_IGNORE_BAD_BLOCKS: y # not recommended for production
+ CC: ${{ matrix.CC }}
+ CXX: ${{ matrix.CXX }}
  run: make -j$(nproc) test
 
 
@@ -66,6 +104,8 @@ jobs:
  name: Ubuntu
 
  coverage_scan:
- needs: call-ubuntu
+ needs: [src_checkers, basic_build]
  uses: ./.github/workflows/scan_coverage.yml
+ secrets:
+ CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
  name: Coverage

.github/workflows/nightly.yml

@@ -35,7 +35,7 @@ jobs:
  TEST_BUILD: [debug, nondebug]
  steps:
  - name: Clone the git repo
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  with:
  fetch-depth: 50
 
@@ -66,7 +66,7 @@ jobs:
  ]
  steps:
  - name: Clone the git repo
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  with:
  fetch-depth: 50
 

.github/workflows/pmem.io_doc_update.yml

@@ -14,7 +14,7 @@ jobs:
  runs-on: ubuntu-latest
  steps:
  - name: Clone the git repo
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
 
  - name: Install dependencies
  run: sudo apt-get install libndctl-dev libdaxctl-dev pandoc

.github/workflows/pmem_benchmark.yml

@@ -0,0 +1,115 @@
+name: PMEM Benchmark
+
+on:
+ workflow_dispatch:
+ inputs:
+ reference_ref:
+ type: string
+ default: stable-2.0
+ rival_ref:
+ type: string
+ default: master
+
+
+jobs:
+ prep_runtime:
+ name: Prepare runtime
+ runs-on: [self-hosted, benchmark]
+ permissions:
+ contents: read
+ steps:
+ - name: Clone the git repo
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 1
+
+ - name: Build
+ run: make -j
+
+
+ prep_contesters:
+ name: Prepare ${{ matrix.ROLE }} (${{ matrix.GITHUB_REF }})
+ runs-on: [self-hosted, benchmark]
+ needs: prep_runtime
+ strategy:
+ matrix:
+ include:
+ - ROLE: reference
+ GITHUB_REF: ${{ inputs.reference_ref }}
+ - ROLE: rival
+ GITHUB_REF: ${{ inputs.rival_ref }}
+ permissions:
+ contents: read
+ env:
+ MANIFEST: ${{ matrix.ROLE }}/manifest.txt
+ steps:
+ - name: Clone the git repo
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ matrix.GITHUB_REF }}
+ fetch-depth: 1
+ path: ${{ matrix.ROLE }}
+
+ - name: Build
+ working-directory: ${{ matrix.ROLE }}
+ run: make -j
+
+ - name: Write the manifest
+ run: |
+ echo "${{ matrix.GITHUB_REF }}" >> $MANIFEST
+ git -C ${{ matrix.ROLE }} rev-parse HEAD >> $MANIFEST
+
+ - name: Archive the manifest
+ uses: actions/upload-artifact@v4
+ with:
+ name: manifest_${{ matrix.ROLE }}
+ path: ${{ env.MANIFEST }}
+
+
+ run:
+ name: Run perf.cfg ${{ matrix.SCENARIO }}
+ runs-on: [self-hosted, benchmark]
+ needs: prep_contesters
+ strategy:
+ matrix:
+ SCENARIO:
+ - obj_tx_alloc_small_v_thread
+ - obj_pmalloc_small_v_threads
+ - obj_rbtree_map_insert
+ - obj_hashmap_tx_map_insert
+ steps:
+ - name: Benchmark
+ uses: ./.github/actions/pmem_benchmark_run
+ with:
+ runtime_dir: ./
+ reference_LIB_PATH: reference/src/nondebug
+ rival_LIB_PATH: rival/src/nondebug
+ config: perf
+ scenario: ${{ matrix.SCENARIO }}
+
+
+ repack:
+ name: Repack
+ runs-on: ubuntu-latest
+ needs: run
+ steps:
+ - name: Download all artifacts
+ uses: actions/download-artifact@v4
+ with:
+ path: csvs
+
+ - name: Combine the manifests
+ env:
+ OUTPUT: csvs/manifest.txt
+ run: |
+ for competitor in reference rival; do
+ echo "$competitor:" >> $OUTPUT
+ cat csvs/manifest_$competitor/manifest.txt >> $OUTPUT
+ echo >> $OUTPUT
+ done
+
+ - name: Upload all as a single artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: perf__all__${{ github.run_id }}
+ path: csvs/**/*

.github/workflows/pmem_ras.yml

@@ -44,7 +44,7 @@ jobs:
 
  steps:
  - name: Clone the git repo
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
 
  # Variables, such as $ras_runner are hidden on the controller platform as environmental variables.
  # 'sed' command is used to filter out IP addresses from the ansible output, it will show up as the 'ras_runner' instead.

.github/workflows/pmem_test_matrix.yml

@@ -9,6 +9,9 @@ on:
  force_enable:
  required: true
  type: string
+ valgrind:
+ required: true
+ type: string
  timeout_minutes:
  required: false
  type: number
@@ -28,11 +31,14 @@ jobs:
  os: [rhel, opensuse]
  build: [debug, nondebug]
 
+
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
 
  - name: Test prepare
  uses: ./.github/actions/pmem_test_prepare
+ with:
+ valgrind: ${{ inputs.valgrind }}
 
  - name: Test run
  uses: ./.github/actions/pmem_test_run