From 463c8eab3583b40f1da3c848066e1c4239b62667 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Mon, 17 Jul 2023 15:39:23 -0400 Subject: [PATCH] common: unify testconfig generation scripts Signed-off-by: Jan Michalski --- .github/workflows/pmem_check.yml | 8 +- .github/workflows/pmem_long.yml | 8 +- .github/workflows/pmem_valgrind.yml | 8 +- utils/create-testconfig.sh | 104 +++++++++++++++++++++++++ utils/docker/configure-tests.sh | 47 ----------- utils/docker/prepare-for-build.sh | 7 +- utils/docker/run-build-package.sh | 7 +- utils/gha-runners/create-testconfig.sh | 53 ------------- 8 files changed, 133 insertions(+), 109 deletions(-) create mode 100755 utils/create-testconfig.sh delete mode 100755 utils/docker/configure-tests.sh delete mode 100755 utils/gha-runners/create-testconfig.sh diff --git a/.github/workflows/pmem_check.yml b/.github/workflows/pmem_check.yml index 67627dc0c3f..5236919b291 100644 --- a/.github/workflows/pmem_check.yml +++ b/.github/workflows/pmem_check.yml @@ -32,7 +32,13 @@ jobs: run: ./$WORKDIR/build-pmdk.sh - name: Create testconfig files - run: ./$WORKDIR/create-testconfig.sh + env: + OUTPUT_DIR: src/test + PMEM_FS_DIR: /mnt/pmem0 + PMEM_FS_DIR_FORCE_PMEM: 0 + force_cacheline: 'True' + force_byte: 'False' + run: ./$WORKDIR/../create-testconfig.sh - name: Run tests (Bash) run: cd src/test/ && ./RUNTESTS.sh -b ${{ matrix.build }} diff --git a/.github/workflows/pmem_long.yml b/.github/workflows/pmem_long.yml index 6324501d182..a1e10746caf 100644 --- a/.github/workflows/pmem_long.yml +++ b/.github/workflows/pmem_long.yml @@ -35,7 +35,13 @@ jobs: run: ./$WORKDIR/build-pmdk.sh - name: Create testconfig file - run: ./$WORKDIR/create-testconfig.sh + env: + OUTPUT_DIR: src/test + PMEM_FS_DIR: /mnt/pmem0 + PMEM_FS_DIR_FORCE_PMEM: 0 + force_cacheline: 'True' + force_byte: 'False' + run: ./$WORKDIR/../create-testconfig.sh - name: Run tests run: cd src/test/ && ./RUNTESTS.${{ matrix.script }} -t long -b ${{ matrix.build }} diff --git a/.github/workflows/pmem_valgrind.yml b/.github/workflows/pmem_valgrind.yml index 87ddb6d6183..bb366d839de 100644 --- a/.github/workflows/pmem_valgrind.yml +++ b/.github/workflows/pmem_valgrind.yml @@ -36,7 +36,13 @@ jobs: run: ./$WORKDIR/build-pmdk.sh - name: Create testconfig file - run: ./$WORKDIR/create-testconfig.sh + env: + OUTPUT_DIR: src/test + PMEM_FS_DIR: /mnt/pmem0 + PMEM_FS_DIR_FORCE_PMEM: 0 + force_cacheline: 'True' + force_byte: 'False' + run: ./$WORKDIR/../create-testconfig.sh - name: Run tests run: cd src/test/ && ./RUNTESTS.${{ matrix.script }} --force-enable ${{ matrix.config }} -b ${{ matrix.build }} diff --git a/utils/create-testconfig.sh b/utils/create-testconfig.sh new file mode 100755 index 00000000000..9e773c4927d --- /dev/null +++ b/utils/create-testconfig.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2023, Intel Corporation + +# +# create-testconfig.sh - creates testconfig files for test execution +# +# Note: It dedicates all Device DAX devices found in the system for testing +# purposes. +# + +# This script makes use of the following variables as they are available in +# the environment: +# - OUTPUT_DIR - the directory where the output testconfig.[sh|py] files are +# about to be written to +# - PMEM_FS_DIR - a directory of a PMem-aware file system +# - PMEM_FS_DIR_FORCE_PMEM - set to 1, if MAP_SYNC in the PMEM_FS_DIR filesystem +# will cause an error. Set to 0 otherwise. +# - force_cacheline - set to True to act in accordance with cache line +# granularity. Set to False otherwise. +# - force_byte - set to True to act in accordance with byte granularity. Set to +# False otherwise. + +# input validation +if [ -z "$OUTPUT_DIR" ]; then + echo "OUTPUT_DIR is empty"; + exit 1; +fi + +if [ -z "$PMEM_FS_DIR" ]; then + echo "PMEM_FS_DIR is empty"; + exit 1; +fi + +if [ -z "$PMEM_FS_DIR_FORCE_PMEM" ]; then + echo "PMEM_FS_DIR_FORCE_PMEM is empty"; + exit 1; +fi + +if [ -z "$force_cacheline" ]; then + echo "force_cacheline is empty"; + exit 1; +fi + +if [ -z "$force_byte" ]; then + echo "force_byte is empty"; + exit 1; +fi + +# converts y/n value to True/False value +function ynToTrueFalse() { + [ "$1" = "y" ] && echo "True" || echo "False" +} + +# static values for both Bash and Python +NON_PMEM_FS_DIR=/dev/shm +KEEP_GOING=y +keep_going=$(ynToTrueFalse $KEEP_GOING) +TEST_BUILD="debug nondebug" +build="['debug', 'nondebug']" +TEST_TIMEOUT=120m +ENABLE_SUDO_TESTS=y +enable_admin_tests=$(ynToTrueFalse $ENABLE_SUDO_TESTS) +TM=1 +[ "$TM" = "1" ] && tm=True || tm=False + +# lookup all Device DAX devices in the system +DEVICE_DAX_PATH= +device_dax_path= +if [ `which ndctl` ]; then + DEVICE_DAX_PATH="$(ndctl list -X | jq -r '.[].daxregion.devices[].chardev' | awk '$0="/dev/"$0' | paste -sd' ')" + device_dax_path="$(ndctl list -X | jq -r '.[].daxregion.devices[].chardev' | awk '$0="'\''/dev/"$0"'\''"' | paste -sd',')" +fi + +# generate Bash's test configuration file +cat << EOL > $OUTPUT_DIR/testconfig.sh +NON_PMEM_FS_DIR=$NON_PMEM_FS_DIR +PMEM_FS_DIR=$PMEM_FS_DIR +PMEM_FS_DIR_FORCE_PMEM=$PMEM_FS_DIR_FORCE_PMEM +DEVICE_DAX_PATH=($DEVICE_DAX_PATH) +KEEP_GOING=$KEEP_GOING +TEST_BUILD="$TEST_BUILD" +TEST_TIMEOUT=$TEST_TIMEOUT +ENABLE_SUDO_TESTS=$ENABLE_SUDO_TESTS +TM=$TM +EOL + +# generate Python's test configuration file +cat << EOL > $OUTPUT_DIR/testconfig.py +config = { + 'page_fs_dir': '${NON_PMEM_FS_DIR}', + 'force_page': False, + 'cacheline_fs_dir': '${PMEM_FS_DIR}', + 'force_cacheline': $force_cacheline, + 'byte_fs_dir': '${PMEM_FS_DIR}', + 'force_byte': $force_byte, + 'device_dax_path' : [$device_dax_path], + 'keep_going': $keep_going, + 'build': $build, + 'timeout': '$TEST_TIMEOUT', + 'enable_admin_tests': $enable_admin_tests, + 'tm': $tm, +} +EOL diff --git a/utils/docker/configure-tests.sh b/utils/docker/configure-tests.sh deleted file mode 100755 index bc061e0e53c..00000000000 --- a/utils/docker/configure-tests.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2016-2023, Intel Corporation - -# -# configure-tests.sh - is called inside a Docker container; configures tests -# for use during build of PMDK project. -# - -set -e - -# Configure tests -cat << EOF > $WORKDIR/src/test/testconfig.sh -LONGDIR=LoremipsumdolorsitametconsecteturadipiscingelitVivamuslacinianibhattortordictumsollicitudinNullamvariusvestibulumligulaetegestaselitsemperidMaurisultriciesligulaeuipsumtinciduntluctusMorbimaximusvariusdolorid -# this path is ~3000 characters long -DIRSUFFIX="$LONGDIR/$LONGDIR/$LONGDIR/$LONGDIR/$LONGDIR" -NON_PMEM_FS_DIR=/tmp -PMEM_FS_DIR=/tmp -PMEM_FS_DIR_FORCE_PMEM=1 -TEST_BUILD="debug nondebug" -ENABLE_SUDO_TESTS=y -TM=1 -EOF - -# Configure python tests -cat << EOF > $WORKDIR/src/test/testconfig.py -config = { - 'unittest_log_level': 1, - 'cacheline_fs_dir': '/tmp', - 'force_cacheline': True, - 'page_fs_dir': '/tmp', - 'force_page': False, - 'byte_fs_dir': '/tmp', - 'force_byte': True, - 'tm': True, - 'test_type': 'check', - 'granularity': 'all', - 'fs_dir_force_pmem': 0, - 'keep_going': False, - 'timeout': '3m', - 'build': ['debug', 'nondebug'], - 'force_enable': None, - 'device_dax_path': [], - 'fail_on_skip': False, - 'enable_admin_tests': True - } -EOF diff --git a/utils/docker/prepare-for-build.sh b/utils/docker/prepare-for-build.sh index 94ecf95ed33..c6be933d1b5 100755 --- a/utils/docker/prepare-for-build.sh +++ b/utils/docker/prepare-for-build.sh @@ -21,5 +21,10 @@ fi # should be preserved KEEP_TEST_CONFIG=${KEEP_TEST_CONFIG:-0} if [[ "$KEEP_TEST_CONFIG" == 0 ]]; then - ./configure-tests.sh + OUTPUT_DIR=$WORKDIR/src/test \ + PMEM_FS_DIR=/dev/shm \ + PMEM_FS_DIR_FORCE_PMEM=1 \ + force_cacheline=True \ + force_byte=True \ + ../create-testconfig.sh fi diff --git a/utils/docker/run-build-package.sh b/utils/docker/run-build-package.sh index 5a52aa41f18..d477f0ec3af 100755 --- a/utils/docker/run-build-package.sh +++ b/utils/docker/run-build-package.sh @@ -63,11 +63,8 @@ pushd $WORKDIR/src/test make -j$(nproc) clobber make -j$(nproc) -# Prepare test config once more. Now, with path to PMDK set in the OS -# (rather than in the git tree) - for testing packages installed in the system. -$SCRIPTSDIR/configure-tests.sh - -# Append variables exclusively for PKG tests +# Append variables with path to PMDK set in the OS (rather than in the git tree) +# - for testing packages installed in the system. case "$OS" in "opensuse/leap" | "rockylinux/rockylinux") PMDK_LIB_PATH_NONDEBUG=/usr/lib64 diff --git a/utils/gha-runners/create-testconfig.sh b/utils/gha-runners/create-testconfig.sh deleted file mode 100755 index ed72a96951c..00000000000 --- a/utils/gha-runners/create-testconfig.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2022-2023, Intel Corporation - -# -# create-testconfig.sh - Script for creating testconfig files for test execution. -# - -# Default location for testconfig.sh -CONF_PATH="src/test" -MOUNT_POINT=${PMDK_MOUNT_POINT:-"/mnt/pmem0"} -NON_PMEM_DIR="/dev/shm" - -# Create config file for unittests. -# We are using ndctl command to gather information about devdaxes, in form known from namespace configuration. -cat >${CONF_PATH}/testconfig.sh <${CONF_PATH}/testconfig.py <