diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml new file mode 100644 index 00000000000..500d6abeea8 --- /dev/null +++ b/.github/workflows/linting.yml @@ -0,0 +1,42 @@ +name: Linting + +# Always run on Pull Requests as then these checks can be marked as required. +on: + push: + branches: + - master + - 'feature/*' + - 'release/*' + pull_request: + +jobs: + # Run isort on the tree. + # This checks .py files only so misses SConstruct and SConscript files are not checked, rather + # for these files check them afterwards. The output-filter will not be installed for this part + # so regressions will be detected but not annotated. + isort: + name: Python isort + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: actions/setup-python@v3 + - uses: isort/isort-action@master + with: + requirementsFiles: "requirements.txt utils/cq/requirements.txt" + - name: Run on SConstruct file. + run: isort --check-only SConstruct + - name: Run on build files. + run: find . -name SConscript | xargs isort --check-only + + log-check: + name: Logging macro checking + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Check DAOS logging macro use. + run: ./utils/cq/d_logging_check.py --github src diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index d02b39405fc..47d6d069c98 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -10,6 +10,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 + - name: Install extra python packages + run: pip install --requirement utils/cq/requirements.txt - name: Run check uses: codespell-project/actions-codespell@master with: diff --git a/.github/workflows/version-checks.yml b/.github/workflows/version-checks.yml new file mode 100644 index 00000000000..11d8abf98f8 --- /dev/null +++ b/.github/workflows/version-checks.yml @@ -0,0 +1,26 @@ +name: Version checking + +on: + push: + branches: + - master + +jobs: + upgrade-check: + name: Check for updates + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + package: [pylint, yamllint, isort, codespell] + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Install extra python packages + run: python3 -m pip install --requirement utils/cq/requirements.txt + - name: Check ${{ matrix.package }} version + run: python -m ${{ matrix.package }} --version | tee -a version-pre + - name: Upgrade + run: pip install --upgrade ${{ matrix.package }} + - name: Check ${{ matrix.package }} for version + run: python -m ${{ matrix.package }} --version | diff version-pre - diff --git a/.github/workflows/yaml.yml b/.github/workflows/yaml.yml index 5f52a7bace4..470db020ae3 100644 --- a/.github/workflows/yaml.yml +++ b/.github/workflows/yaml.yml @@ -11,6 +11,7 @@ on: paths: - '**/*.yaml' - '**/*.yml' + - utils/cq/requirements.txt jobs: yaml-lint: @@ -25,5 +26,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: '3' + - name: Install extra python packages + run: pip install --requirement utils/cq/requirements.txt - name: Run check run: yamllint --format github . diff --git a/Jenkinsfile b/Jenkinsfile index 28f8d1f56d3..4536019bc48 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -155,6 +155,21 @@ void fixup_rpmlintrc() { writeFile(file: 'utils/rpms/daos.rpmlintrc', text: content) } +String vm9_label(String distro) { + return cachedCommitPragma(pragma: distro + '-VM9-label', + def_val: cachedCommitPragma(pragma: 'VM9-label', + def_val: params.FUNCTIONAL_VM_LABEL)) +} + +void rpm_test_post(String stage_name, String node) { + sh label: 'Fetch and stage artifacts', + script: 'hostname; ssh -i ci_key jenkins@' + node + ' ls -ltar /tmp; mkdir -p "' + env.STAGE_NAME + '/" && ' + + 'scp -i ci_key jenkins@' + node + ':/tmp/{{suite_dmg,daos_{server_helper,{control,agent}}}.log,daos_server.log.*} "' + + env.STAGE_NAME + '/"' + archiveArtifacts artifacts: env.STAGE_NAME + '/**' + job_status_update() +} + pipeline { agent { label 'lightweight' } @@ -516,7 +531,7 @@ pipeline { filename 'packaging/Dockerfile.mockbuild' dir 'utils/rpms' label 'docker_runner' - additionalBuildArgs dockerBuildArgs() + additionalBuildArgs dockerBuildArgs() + '--build-arg FVERSION=38' args '--cap-add=SYS_ADMIN' } } @@ -543,7 +558,7 @@ pipeline { } } } - stage('Build RPM on Leap 15.4') { + stage('Build RPM on Leap 15.5') { when { beforeAgent true expression { !skipStage() } @@ -553,7 +568,7 @@ pipeline { filename 'packaging/Dockerfile.mockbuild' dir 'utils/rpms' label 'docker_runner' - additionalBuildArgs dockerBuildArgs() + '--build-arg FVERSION=37' + additionalBuildArgs dockerBuildArgs() + '--build-arg FVERSION=38' args '--cap-add=SYS_ADMIN' } } @@ -700,7 +715,7 @@ pipeline { } } } - stage('Build on Leap 15.4 with Intel-C and TARGET_PREFIX') { + stage('Build on Leap 15.5 with Intel-C and TARGET_PREFIX') { when { beforeAgent true expression { !skipStage() } @@ -744,7 +759,7 @@ pipeline { expression { !skipStage() } } parallel { - stage('Unit Test on EL 8') { + stage('Unit Test on EL 8.8') { when { beforeAgent true expression { !skipStage() } @@ -766,7 +781,7 @@ pipeline { } } } - stage('Unit Test bdev on EL 8') { + stage('Unit Test bdev on EL 8.8') { when { beforeAgent true expression { !skipStage() } @@ -788,7 +803,7 @@ pipeline { } } } - stage('NLT on EL 8') { + stage('NLT on EL 8.8') { when { beforeAgent true expression { !skipStage() } @@ -825,7 +840,7 @@ pipeline { } } } - stage('Unit Test Bullseye on EL 8') { + stage('Unit Test Bullseye on EL 8.8') { when { beforeAgent true expression { !skipStage() } @@ -852,8 +867,8 @@ pipeline { job_status_update() } } - } // stage('Unit test Bullseye on EL 8') - stage('Unit Test with memcheck on EL 8') { + } // stage('Unit test Bullseye on EL 8.8') + stage('Unit Test with memcheck on EL 8.8') { when { beforeAgent true expression { !skipStage() } @@ -877,8 +892,8 @@ pipeline { job_status_update() } } - } // stage('Unit Test with memcheck on EL 8') - stage('Unit Test bdev with memcheck on EL 8') { + } // stage('Unit Test with memcheck on EL 8.8') + stage('Unit Test bdev with memcheck on EL 8.8') { when { beforeAgent true expression { !skipStage() } @@ -911,7 +926,7 @@ pipeline { expression { !skipStage() } } parallel { - stage('Functional on EL 8 with Valgrind') { + stage('Functional on EL 8.8 with Valgrind') { when { beforeAgent true expression { !skipStage() } @@ -932,14 +947,14 @@ pipeline { job_status_update() } } - } // stage('Functional on EL 8 with Valgrind') - stage('Functional on EL 8') { + } // stage('Functional on EL 8.8 with Valgrind') + stage('Functional on EL 8.8') { when { beforeAgent true expression { !skipStage() } } agent { - label cachedCommitPragma(pragma: 'EL8-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL) + label vm9_label('EL8') } steps { job_step_update( @@ -954,14 +969,14 @@ pipeline { job_status_update() } } - } // stage('Functional on EL 8') - stage('Functional on Leap 15.4') { + } // stage('Functional on EL 8.8') + stage('Functional on Leap 15.5') { when { beforeAgent true expression { !skipStage() } } agent { - label cachedCommitPragma(pragma: 'Leap15-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL) + label vm9_label('Leap15') } steps { job_step_update( @@ -976,14 +991,14 @@ pipeline { job_status_update() } } // post - } // stage('Functional on Leap 15.4') + } // stage('Functional on Leap 15.5') stage('Functional on Ubuntu 20.04') { when { beforeAgent true expression { !skipStage() } } agent { - label cachedCommitPragma(pragma: 'Ubuntu-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL) + label vm9_label('Ubuntu') } steps { job_step_update( @@ -999,44 +1014,7 @@ pipeline { } } // post } // stage('Functional on Ubuntu 20.04') - stage('Scan EL 8 RPMs') { - when { - beforeAgent true - expression { !skipStage() } - } - agent { - dockerfile { - filename 'ci/docker/Dockerfile.maldet.el.8' - label 'docker_runner' - additionalBuildArgs dockerBuildArgs() + - " -t ${sanitized_JOB_NAME}-el8 " + - ' --build-arg REPOS="' + prRepos() + '"' + - ' --build-arg BUILD_URL="' + env.BUILD_URL + '"' - } - } - steps { - job_step_update( - runTest(script: 'export DAOS_PKG_VERSION=' + - daosPackagesVersion(next_version) + '\n' + - 'utils/scripts/helpers/scan_daos_maldet.sh', - junit_files: 'maldetect_el8.xml', - failure_artifacts: env.STAGE_NAME, - ignore_failure: true, - description: env.STAGE_NAME, - context: 'test/' + env.STAGE_NAME)) - } - post { - always { - junit 'maldetect_el8.xml' - archiveArtifacts artifacts: 'maldetect_el8.xml' - job_status_update() - // Force a job failure if anything was found - sh label: 'Check if anything was found.', - script: '! grep "/dev/null | grep ..\*); then + DNF_REPO_ARGS+=" --enablerepo=$repo" + else + local repo_url="${REPOSITORY_URL}${add_repo}" + local repo_name + repo_name=$(url_to_repo "$repo_url") + if ! dnf -y repolist | grep "$repo_name"; then + dnf -y config-manager --add-repo="${repo_url}" >&2 + if ! $gpg_check; then + disable_gpg_check "$add_repo" >&2 + fi + fi + DNF_REPO_ARGS+=" --enablerepo=$repo_name" + fi +} + +add_group_repo() { + local match="$1" + + add_repo "$match" "$DAOS_STACK_GROUP_REPO" + group_repo_post +} + +add_local_repo() { + add_repo 'argobots' "$DAOS_STACK_LOCAL_REPO" false +} + +disable_gpg_check() { + local url="$1" + + repo="$(url_to_repo "$url")" + # bug in EL7 DNF: this needs to be enabled before it can be disabled + dnf -y config-manager --save --setopt="$repo".gpgcheck=1 + dnf -y config-manager --save --setopt="$repo".gpgcheck=0 + # but even that seems to be not enough, so just brute-force it + if [ -d /etc/yum.repos.d ] && + ! grep gpgcheck /etc/yum.repos.d/"$repo".repo; then + echo "gpgcheck=0" >> /etc/yum.repos.d/"$repo".repo + fi +} + +dump_repos() { + for file in "$REPOS_DIR"/*.repo; do + echo "---- $file ----" + cat "$file" + done +} retry_dnf() { local monitor_threshold="$1" shift @@ -259,7 +330,7 @@ post_provision_config_nodes() { if $CONFIG_POWER_ONLY; then rm -f "$REPOS_DIR"/*.hpdd.intel.com_job_daos-stack_job_*_job_*.repo - time dnf -y erase fio fuse ior-hpc mpich-autoload \ + time dnf -y erase fio fuse ior-hpc mpich-autoload \ ompi argobots cart daos daos-client dpdk \ fuse-libs libisa-l libpmemobj mercury mpich \ pmix protobuf-c spdk libfabric libpmem \ @@ -267,7 +338,41 @@ post_provision_config_nodes() { slurm-example-configs slurmctld slurm-slurmmd fi - lsb_release -a + cat /etc/os-release + + if lspci | grep "ConnectX-6" && ! grep MOFED_VERSION /etc/do-release; then + # Remove OPA and install MOFED + install_mofed + fi + + local repos_added=() + local repo + local inst_repos=() + # shellcheck disable=SC2153 + read -ra inst_repos <<< "$INST_REPOS" + for repo in "${inst_repos[@]}"; do + branch="master" + build_number="lastSuccessfulBuild" + if [[ $repo = *@* ]]; then + branch="${repo#*@}" + repo="${repo%@*}" + if [[ \ ${repos_added[*]+${repos_added[*]}}\ = *\ ${repo}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo") + if [[ $branch = *:* ]]; then + build_number="${branch#*:}" + branch="${branch%:*}" + fi + fi + local repo_url="${ARTIFACTS_URL:-${JENKINS_URL}job/}"daos-stack/job/"$repo"/job/"${branch//\//%252F}"/"$build_number"/artifact/artifacts/$DISTRO_NAME/ + dnf -y config-manager --add-repo="$repo_url" + repo="$(url_to_repo "$repo_url")" + # PR-repos: should always be able to upgrade modular packages + dnf -y config-manager --save --setopt "$repo.module_hotfixes=true" "$repo" + disable_gpg_check "$repo_url" + done # start with everything fully up-to-date # all subsequent package installs beyond this will install the newest packages @@ -283,30 +388,8 @@ post_provision_config_nodes() { return 1 fi - if lspci | grep "ConnectX-6" && ! grep MOFED_VERSION /etc/do-release; then - # Remove OPA and install MOFED - install_mofed - fi - - if [ -n "$INST_REPOS" ]; then - local repo - for repo in $INST_REPOS; do - branch="master" - build_number="lastSuccessfulBuild" - if [[ $repo = *@* ]]; then - branch="${repo#*@}" - repo="${repo%@*}" - if [[ $branch = *:* ]]; then - build_number="${branch#*:}" - branch="${branch%:*}" - fi - fi - local repo_url="${JENKINS_URL}"job/daos-stack/job/"${repo}"/job/"${branch//\//%252F}"/"${build_number}"/artifact/artifacts/$DISTRO_NAME/ - dnf -y config-manager --add-repo="${repo_url}" - disable_gpg_check "$repo_url" - done - fi - inst_rpms=() + local inst_rpms=() + # shellcheck disable=SC2153 if [ -n "$INST_RPMS" ]; then eval "inst_rpms=($INST_RPMS)" time dnf -y erase "${inst_rpms[@]}" @@ -358,7 +441,7 @@ setenv MPI_HOME /usr/mpi/gcc/openmpi-$version EOF printf 'MOFED_VERSION=%s\n' "$MLNX_VER_NUM" >> /etc/do-release - fi + fi distro_custom diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 1b205946f6b..8d70a0c1a51 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -2,77 +2,6 @@ set -eux -# functions common to more than one distro specific provisioning -url_to_repo() { - local url="$1" - - local repo=${url#*://} - repo="${repo//%/}" - repo="${repo//\//_}" - - echo "$repo" -} - -add_repo() { - local match="$1" - local add_repo="$2" - local gpg_check="${3:-true}" - - if [ -z "$match" ]; then - # we cannot try to add a repo that has no match - return - fi - - local repo - # see if a package we know is in the repo is present - if repo=$(dnf -y repoquery --qf "%{repoid}" "$1" 2>/dev/null | grep ..\*); then - DNF_REPO_ARGS+=" --enablerepo=$repo" - else - local repo_url="${REPOSITORY_URL}${add_repo}" - local repo_name - repo_name=$(url_to_repo "$repo_url") - if ! dnf -y repolist | grep "$repo_name"; then - dnf -y config-manager --add-repo="${repo_url}" >&2 - if ! $gpg_check; then - disable_gpg_check "$add_repo" >&2 - fi - fi - DNF_REPO_ARGS+=" --enablerepo=$repo_name" - fi -} - -add_group_repo() { - local match="$1" - - add_repo "$match" "$DAOS_STACK_GROUP_REPO" - group_repo_post -} - -add_local_repo() { - add_repo 'argobots' "$DAOS_STACK_LOCAL_REPO" false -} - -disable_gpg_check() { - local url="$1" - - repo="$(url_to_repo "$url")" - # bug in EL7 DNF: this needs to be enabled before it can be disabled - dnf -y config-manager --save --setopt="$repo".gpgcheck=1 - dnf -y config-manager --save --setopt="$repo".gpgcheck=0 - # but even that seems to be not enough, so just brute-force it - if [ -d /etc/yum.repos.d ] && - ! grep gpgcheck /etc/yum.repos.d/"$repo".repo; then - echo "gpgcheck=0" >> /etc/yum.repos.d/"$repo".repo - fi -} - -dump_repos() { - for file in "$REPOS_DIR"/*.repo; do - echo "---- $file ----" - cat "$file" - done -} - env > /root/last_run-env.txt if ! grep ":$MY_UID:" /etc/group; then groupadd -g "$MY_UID" jenkins diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index c7b27473593..3e92bb7b703 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -1,25 +1,31 @@ #!/bin/bash +. /etc/os-release + YUM=dnf -id="$(lsb_release -si)" -if [ "$id" = "CentOS" ] || - [ "$id" = "AlmaLinux" ] || - [ "$id" = "Rocky" ] || - [ "$id" = "RedHatEnterpriseServer" ]; then - if [[ $(lsb_release -sr) = 8* ]]; then - OPENMPI_RPM=openmpi - OPENMPI=mpi/openmpi-x86_64 - else +case "$ID_LIKE" in + *rhel*) + if [[ $VERSION_ID = [89].* ]]; then + OPENMPI_RPM=openmpi + OPENMPI=mpi/openmpi-x86_64 + else + OPENMPI_RPM=openmpi3 + OPENMPI=mpi/openmpi3-x86_64 + fi + ;; + *suse*) OPENMPI_RPM=openmpi3 - OPENMPI=mpi/openmpi3-x86_64 - fi -elif [ "$(lsb_release -si)" = "openSUSE" ]; then - OPENMPI_RPM=openmpi3 - OPENMPI=gnu-openmpi + OPENMPI=gnu-openmpi + export MODULEPATH=/usr/share/modules + ;; +esac + +if [ -n "$DAOS_PKG_VERSION" ]; then + DAOS_PKG_VERSION="-${DAOS_PKG_VERSION}" fi set -uex -sudo $YUM -y install daos-client-"${DAOS_PKG_VERSION}" +sudo $YUM -y install daos-client"$DAOS_PKG_VERSION" if rpm -q daos-server; then echo "daos-server RPM should not be installed as a dependency of daos-client" exit 1 @@ -29,9 +35,9 @@ if ! sudo $YUM -y history undo last; then $YUM history exit 1 fi -sudo $YUM -y erase $OPENMPI_RPM -sudo $YUM -y install daos-client-tests-"${DAOS_PKG_VERSION}" -if rpm -q $OPENMPI_RPM; then +sudo $YUM -y erase "$OPENMPI_RPM" +sudo $YUM -y install daos-client-tests"$DAOS_PKG_VERSION" +if rpm -q "$OPENMPI_RPM"; then echo "$OPENMPI_RPM RPM should not be installed as a dependency of daos-client-tests" exit 1 fi @@ -44,8 +50,8 @@ if ! sudo $YUM -y history undo last; then $YUM history exit 1 fi -sudo $YUM -y install daos-server-tests-"${DAOS_PKG_VERSION}" -if rpm -q $OPENMPI_RPM; then +sudo $YUM -y install daos-server-tests"$DAOS_PKG_VERSION" +if rpm -q "$OPENMPI_RPM"; then echo "$OPENMPI_RPM RPM should not be installed as a dependency of daos-server-tests" exit 1 fi @@ -58,7 +64,7 @@ if ! sudo $YUM -y history undo last; then $YUM history exit 1 fi -sudo $YUM -y install --exclude ompi daos-client-tests-openmpi-"${DAOS_PKG_VERSION}" +sudo $YUM -y install --exclude ompi daos-client-tests-openmpi"$DAOS_PKG_VERSION" if ! rpm -q daos-client; then echo "daos-client RPM should be installed as a dependency of daos-client-tests-openmpi" exit 1 @@ -76,13 +82,13 @@ if ! sudo $YUM -y history undo last; then $YUM history exit 1 fi -sudo $YUM -y install daos-server-"${DAOS_PKG_VERSION}" +sudo $YUM -y install daos-server"$DAOS_PKG_VERSION" if rpm -q daos-client; then echo "daos-client RPM should not be installed as a dependency of daos-server" exit 1 fi -sudo $YUM -y install --exclude ompi daos-client-tests-openmpi-"${DAOS_PKG_VERSION}" +sudo $YUM -y install --exclude ompi daos-client-tests-openmpi"$DAOS_PKG_VERSION" me=$(whoami) for dir in server agent; do @@ -93,42 +99,63 @@ done sudo mkdir /tmp/daos_sockets sudo chmod 0755 /tmp/daos_sockets sudo chown "$me:$me" /tmp/daos_sockets -sudo mkdir -p /mnt/daos -sudo mount -t tmpfs -o size=16777216k tmpfs /mnt/daos FTEST=/usr/lib/daos/TESTING/ftest sudo PYTHONPATH="$FTEST/util" \ $FTEST/config_file_gen.py -n "$HOSTNAME" \ -a /etc/daos/daos_agent.yml \ -s /etc/daos/daos_server.yml +sudo bash -c 'echo "system_ram_reserved: 4" >> /etc/daos/daos_server.yml' sudo PYTHONPATH="$FTEST/util" \ $FTEST/config_file_gen.py -n "$HOSTNAME" \ - -d /etc/daos/daos.yml + -d /etc/daos/daos_control.yml cat /etc/daos/daos_server.yml cat /etc/daos/daos_agent.yml -cat /etc/daos/daos.yml -if ! module load $OPENMPI; then +cat /etc/daos/daos_control.yml +if ! module load "$OPENMPI"; then echo "Unable to load OpenMPI module: $OPENMPI" module avail module list exit 1 fi -coproc SERVER { exec daos_server --debug start -t 1 --recreate-superblocks; } 2>&1 +export POOL_NVME_SIZE=0 + +coproc SERVER { exec daos_server --debug start -t 1; } 2>&1 trap 'set -x; kill -INT $SERVER_PID' EXIT line="" -while [[ "$line" != *started\ on\ rank\ 0* ]]; do - if ! read -r -t 60 line <&"${SERVER[0]}"; then - rc=${PIPESTATUS[0]} - if [ "$rc" = "142" ]; then - echo "Timed out waiting for output from the server" - else - echo "Error reading the output from the server: $rc" - fi - exit "$rc" - fi - echo "Server stdout: $line" +stdout=() +deadline=$((SECONDS+300)) +while [[ $line != *started\ on\ rank\ 0* ]] && [ "$SECONDS" -lt "$deadline" ]; do + if ! read -r -t 60 line <&"${SERVER[0]}"; then + rc=${PIPESTATUS[0]} + if [ "$rc" = "142" ]; then + echo "Timed out waiting for output from the server" + else + echo "Error reading the output from the server: $rc" + fi + echo "Server output:" + export IFS=$'\n' + echo "${stdout[*]}" + exit "$rc" + fi + echo "Server stdout: $line" + stdout+=("$line") + if [[ $line == SCM\ format\ required\ on\ instance\ * ]]; then + dmg storage format -l "$HOSTNAME" --force + if ! dmg system query -v; then + sleep 5 + dmg system query -v || true + fi + fi done +if [ "$SECONDS" -ge "$deadline" ]; then + echo "Timed out waiting for server to start" + echo "Server output:" + export IFS=$'\n' + echo "${stdout[*]}" + exit 1 +fi echo "Server started!" coproc AGENT { exec daos_agent --debug; } 2>&1 trap 'set -x; kill -INT $AGENT_PID $SERVER_PID' EXIT diff --git a/debian/changelog b/debian/changelog index 704a3bf07de..d9d5e80c087 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,17 @@ +daos (2.4.1-1) unstable; urgency=medium + + [ Phillip Henderson ] + * First release candidate for 2.4.1 + + -- Phillip Henderson Fri, 08 Dec 2023 16:12:00 -0500 + +daos (2.4.0-5) unstable; urgency=medium + [ Brian J. Murrell ] + * Update for EL 8.8 and Leap 15.5 + * Update raft to 0.10.1-2.411.gefa15f4 + + -- Brian J. Murrell Wed, 06 Dec 2023 08:54:56 -0500 + daos (2.4.0-4) unstable; urgency=medium [ Jerome Soumagne ] * Bump mercury min version to 2.3.1 diff --git a/debian/control b/debian/control index 6afa7bf47af..c717114254d 100644 --- a/debian/control +++ b/debian/control @@ -29,7 +29,7 @@ Build-Depends: debhelper (>= 10), libboost-dev, libspdk-dev (>= 22.01.2), libipmctl-dev, - libraft-dev (= 0.10.1-1408.g9524cdb), + libraft-dev (= 0.10.1-1411.gefa15f4), python3-tabulate, liblz4-dev, libcapstone-dev @@ -201,4 +201,3 @@ Description: The Distributed Asynchronous Object Storage (DAOS) is an open-sourc to optimize performance and cost. . This package contains the DAOS administrative tools (e.g. dmg). - diff --git a/ftest.sh b/ftest.sh index 4963cec765a..94661790c5c 100755 --- a/ftest.sh +++ b/ftest.sh @@ -29,9 +29,7 @@ LAUNCH_OPT_ARGS="${3:-}" # Add the missing '--nvme' argument identifier for backwards compatibility with # the 'auto:Optane' optional argument specified without the identifier. -if [[ "${LAUNCH_OPT_ARGS}" == "auto:-3DNAND" ]]; then - LAUNCH_OPT_ARGS="--nvme=${LAUNCH_OPT_ARGS}" -fi +LAUNCH_OPT_ARGS="$(echo "$LAUNCH_OPT_ARGS" | sed -e 's/^/ /' -e 's/ \(auto:-3DNAND\)/--nvme=\1/' -e 's/^ *//')" # For nodes that are only rebooted between CI nodes left over mounts # need to be cleaned up. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..15ea695e048 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,5 @@ +[tool.isort] +supported_extensions = ["py"] +skip = [".git/", "src/rdb/raft", "build", "install", "venv", "src/control/vendor/"] +line_length = 99 +skip_gitignore = true diff --git a/requirements.txt b/requirements.txt index 769762707f0..93ea995a9e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -# Requirements which are not versioned. +# Packages required to build/test DAOS. defusedxml distro jira @@ -8,6 +8,6 @@ ninja pyelftools pyxattr pyyaml -scons # Works around a bug in scons on EL 8. +scons tabulate wheel diff --git a/site_scons/components/__init__.py b/site_scons/components/__init__.py index 1e764464ec3..76363ed7af0 100644 --- a/site_scons/components/__init__.py +++ b/site_scons/components/__init__.py @@ -21,9 +21,10 @@ """Defines common components used by HPDD projects""" import platform + import distro -from SCons.Script import GetOption from prereq_tools import GitRepoRetriever +from SCons.Script import GetOption # Check if this is an ARM platform PROCESSOR = platform.machine() diff --git a/site_scons/env_modules.py b/site_scons/env_modules.py index 97dba45feb2..ed2b6b295c0 100644 --- a/site_scons/env_modules.py +++ b/site_scons/env_modules.py @@ -19,12 +19,13 @@ # SOFTWARE. """Wrapper for Modules so we can load an MPI before builds or tests""" -import os -import sys import errno -import subprocess # nosec +import os import shutil +import subprocess # nosec +import sys from subprocess import PIPE, Popen # nosec + import distro diff --git a/site_scons/prereq_tools/base.py b/site_scons/prereq_tools/base.py index 8ba3833f686..e1d86d54899 100644 --- a/site_scons/prereq_tools/base.py +++ b/site_scons/prereq_tools/base.py @@ -20,28 +20,21 @@ # -*- coding: utf-8 -*- """Classes for building external prerequisite components""" -# pylint: disable=too-many-lines -import os -from copy import deepcopy -import sys -import json +import configparser import datetime -import traceback import errno +import json +# pylint: disable=too-many-lines +import os import shutil import subprocess # nosec -import configparser -from SCons.Variables import BoolVariable -from SCons.Variables import EnumVariable -from SCons.Variables import ListVariable -from SCons.Variables import PathVariable -from SCons.Script import Dir -from SCons.Script import Exit -from SCons.Script import GetOption -from SCons.Script import SetOption -from SCons.Script import WhereIs -from SCons.Script import BUILD_TARGETS +import sys +import traceback +from copy import deepcopy + from SCons.Errors import InternalError +from SCons.Script import BUILD_TARGETS, Dir, Exit, GetOption, SetOption, WhereIs +from SCons.Variables import BoolVariable, EnumVariable, ListVariable, PathVariable class DownloadFailure(Exception): diff --git a/site_scons/site_tools/compiler_setup.py b/site_scons/site_tools/compiler_setup.py index 163acf2c406..584eb4e997f 100644 --- a/site_scons/site_tools/compiler_setup.py +++ b/site_scons/site_tools/compiler_setup.py @@ -1,8 +1,6 @@ """Common DAOS library for setting up the compiler""" -from SCons.Script import GetOption, Exit -from SCons.Script import Configure - +from SCons.Script import Configure, Exit, GetOption DESIRED_FLAGS = ['-Wno-gnu-designator', '-Wno-missing-braces', diff --git a/site_scons/site_tools/daos_builder.py b/site_scons/site_tools/daos_builder.py index 36676952ffd..4f06c9c93b2 100644 --- a/site_scons/site_tools/daos_builder.py +++ b/site_scons/site_tools/daos_builder.py @@ -1,13 +1,9 @@ """Common DAOS build functions""" import os -from SCons.Subst import Literal -from SCons.Script import Dir -from SCons.Script import GetOption -from SCons.Script import WhereIs -from SCons.Script import Depends -from SCons.Script import Exit from env_modules import load_mpi +from SCons.Script import Depends, Dir, Exit, GetOption, WhereIs +from SCons.Subst import Literal libraries = {} missing = set() diff --git a/site_scons/site_tools/doneapi.py b/site_scons/site_tools/doneapi.py index b57c569d461..b9938b5a197 100644 --- a/site_scons/site_tools/doneapi.py +++ b/site_scons/site_tools/doneapi.py @@ -3,13 +3,13 @@ Hack to support oneapi version of Intel compilers """ -import sys import os +import sys +import SCons.Errors import SCons.Tool.gcc import SCons.Util import SCons.Warnings -import SCons.Errors # pylint: disable=too-few-public-methods diff --git a/site_scons/site_tools/extra/__init__.py b/site_scons/site_tools/extra/__init__.py index 2003b75110e..f44a9530de1 100644 --- a/site_scons/site_tools/extra/__init__.py +++ b/site_scons/site_tools/extra/__init__.py @@ -7,4 +7,4 @@ # pylint: disable=unused-import -from .extra import generate, exists # noqa: F401 +from .extra import exists, generate # noqa: F401 diff --git a/site_scons/site_tools/extra/extra.py b/site_scons/site_tools/extra/extra.py index b3054b0fc1f..e7d6c38bfee 100755 --- a/site_scons/site_tools/extra/extra.py +++ b/site_scons/site_tools/extra/extra.py @@ -9,9 +9,9 @@ This is used by scons to reformat automatically generated header files to be readable, but also outside of scons by the clang-format commit hook to check the version. """ -import subprocess # nosec -import re import os +import re +import subprocess # nosec import sys from SCons.Builder import Builder diff --git a/site_scons/site_tools/go_builder.py b/site_scons/site_tools/go_builder.py index 6829f688f06..b3706a8976d 100644 --- a/site_scons/site_tools/go_builder.py +++ b/site_scons/site_tools/go_builder.py @@ -1,11 +1,11 @@ """DAOS functions for building go""" -import subprocess # nosec B404 +import json import os import re -import json +import subprocess # nosec B404 -from SCons.Script import Configure, GetOption, Scanner, Glob, Exit, File +from SCons.Script import Configure, Exit, File, GetOption, Glob, Scanner GO_COMPILER = 'go' MIN_GO_VERSION = '1.18.0' diff --git a/site_scons/site_tools/protoc/__init__.py b/site_scons/site_tools/protoc/__init__.py index dea641af3d3..239ab73422c 100644 --- a/site_scons/site_tools/protoc/__init__.py +++ b/site_scons/site_tools/protoc/__init__.py @@ -20,7 +20,9 @@ # SOFTWARE. import os + import SCons.Builder + # pylint: disable=too-few-public-methods,missing-class-docstring diff --git a/site_scons/site_tools/stack_analyzer.py b/site_scons/site_tools/stack_analyzer.py index 46830086f5a..11fe8af1734 100644 --- a/site_scons/site_tools/stack_analyzer.py +++ b/site_scons/site_tools/stack_analyzer.py @@ -5,9 +5,10 @@ Analyze stack usage output """ -import os import argparse import atexit +import os + from SCons.Script import Exit diff --git a/src/bio/bio_xstream.c b/src/bio/bio_xstream.c index c7819a84318..6242ad6f4ef 100644 --- a/src/bio/bio_xstream.c +++ b/src/bio/bio_xstream.c @@ -958,8 +958,8 @@ init_bio_bdevs(struct bio_xs_context *ctxt) D_ASSERT(!is_server_started()); if (spdk_bdev_first() == NULL) { - D_ERROR("No SPDK bdevs found!"); - rc = -DER_NONEXIST; + D_ERROR("No SPDK bdevs found!\n"); + return -DER_NONEXIST; } for (bdev = spdk_bdev_first(); bdev != NULL; bdev = spdk_bdev_next(bdev)) { diff --git a/src/cart/SConscript b/src/cart/SConscript index e44751a06b0..9b5b2c8ecef 100644 --- a/src/cart/SConscript +++ b/src/cart/SConscript @@ -4,6 +4,7 @@ # """Build CaRT components""" from datetime import date + import SCons.Action SRC = ['crt_bulk.c', 'crt_context.c', 'crt_corpc.c', diff --git a/src/cart/crt_group.c b/src/cart/crt_group.c index cb34c06160c..f02ebdc96b8 100644 --- a/src/cart/crt_group.c +++ b/src/cart/crt_group.c @@ -2404,10 +2404,11 @@ grp_regen_linear_list(struct crt_grp_priv *grp_priv) /* If group size changed - reallocate the list */ if (!linear_list->rl_ranks || linear_list->rl_nr != grp_priv->gp_size) { - linear_list = d_rank_list_realloc(linear_list, - grp_priv->gp_size); - if (linear_list == NULL) - return -DER_NOMEM; + int rc; + + rc = d_rank_list_resize(linear_list, grp_priv->gp_size); + if (rc != 0) + return rc; } index = 0; @@ -2460,9 +2461,9 @@ grp_add_to_membs_list(struct crt_grp_priv *grp_priv, d_rank_t rank, uint64_t inc first = membs->rl_nr; new_amount = first + RANK_LIST_REALLOC_SIZE; - membs = d_rank_list_realloc(membs, new_amount); - if (membs == NULL) - D_GOTO(out, rc = -DER_NOMEM); + rc = d_rank_list_resize(membs, new_amount); + if (rc != 0) + D_GOTO(out, rc); for (i = first; i < first + RANK_LIST_REALLOC_SIZE; i++) { membs->rl_ranks[i] = CRT_NO_RANK; diff --git a/src/cart/crt_hg.c b/src/cart/crt_hg.c index 299088524eb..2b74f882bc1 100644 --- a/src/cart/crt_hg.c +++ b/src/cart/crt_hg.c @@ -874,6 +874,9 @@ crt_hg_class_init(int provider, int idx, bool primary, hg_class_t **ret_hg_class if (prov_data->cpg_max_unexp_size > 0) init_info.na_init_info.max_unexpected_size = prov_data->cpg_max_unexp_size; + init_info.request_post_init = crt_gdata.cg_post_init; + init_info.request_post_incr = crt_gdata.cg_post_incr; + hg_class = HG_Init_opt(info_string, crt_is_service(), &init_info); if (hg_class == NULL) { D_ERROR("Could not initialize HG class.\n"); diff --git a/src/cart/crt_hg.h b/src/cart/crt_hg.h index 48ea4c74f41..dacb9fed577 100644 --- a/src/cart/crt_hg.h +++ b/src/cart/crt_hg.h @@ -27,6 +27,10 @@ /** number of prepost HG handles when enable pool */ #define CRT_HG_POOL_PREPOST_NUM (16) +/** default values for init / incr to prepost handles */ +#define CRT_HG_POST_INIT (512) +#define CRT_HG_POST_INCR (512) + struct crt_rpc_priv; struct crt_common_hdr; struct crt_corpc_hdr; diff --git a/src/cart/crt_hg_proc.c b/src/cart/crt_hg_proc.c index 6001c04b1a3..47d003ee822 100644 --- a/src/cart/crt_hg_proc.c +++ b/src/cart/crt_hg_proc.c @@ -254,6 +254,47 @@ crt_proc_d_iov_t(crt_proc_t proc, crt_proc_op_t proc_op, d_iov_t *div) return rc; } +int +crt_proc_d_sg_list_t(crt_proc_t proc, crt_proc_op_t proc_op, d_sg_list_t *p) +{ + int i; + int rc; + + if (FREEING(proc_op)) { + /* NB: don't need free in crt_proc_d_iov_t() */ + D_FREE(p->sg_iovs); + return 0; + } + + rc = crt_proc_uint32_t(proc, proc_op, &p->sg_nr); + if (unlikely(rc)) + return rc; + + rc = crt_proc_uint32_t(proc, proc_op, &p->sg_nr_out); + if (unlikely(rc)) + return rc; + + if (p->sg_nr == 0) + return 0; + + if (DECODING(proc_op)) { + D_ALLOC_ARRAY(p->sg_iovs, p->sg_nr); + if (p->sg_iovs == NULL) + return -DER_NOMEM; + } + + for (i = 0; i < p->sg_nr; i++) { + rc = crt_proc_d_iov_t(proc, proc_op, &p->sg_iovs[i]); + if (unlikely(rc)) { + if (DECODING(proc_op)) + D_FREE(p->sg_iovs); + return rc; + } + } + + return rc; +} + static inline int crt_proc_corpc_hdr(crt_proc_t proc, struct crt_corpc_hdr *hdr) { diff --git a/src/cart/crt_init.c b/src/cart/crt_init.c index 988e35db701..308b9c946f4 100644 --- a/src/cart/crt_init.c +++ b/src/cart/crt_init.c @@ -49,6 +49,7 @@ crt_lib_init(void) crt_gdata.cg_rpcid = start_rpcid; crt_gdata.cg_num_cores = sysconf(_SC_NPROCESSORS_ONLN); + crt_gdata.cg_iv_inline_limit = 19456; /* 19KB */ } /* Library deinit */ @@ -63,16 +64,39 @@ dump_envariables(void) { int i; char *val; - char *envars[] = {"D_PROVIDER", "D_INTERFACE", "D_DOMAIN", "D_PORT", - "CRT_PHY_ADDR_STR", "D_LOG_STDERR_IN_LOG", "D_LOG_SIZE", - "D_LOG_FILE", "D_LOG_FILE_APPEND_PID", "D_LOG_MASK", "DD_MASK", - "DD_STDERR", "DD_SUBSYS", "CRT_TIMEOUT", "CRT_ATTACH_INFO_PATH", - "OFI_PORT", "OFI_INTERFACE", "OFI_DOMAIN", "CRT_CREDIT_EP_CTX", - "CRT_CTX_SHARE_ADDR", "CRT_CTX_NUM", "D_FI_CONFIG", - "FI_UNIVERSE_SIZE", "CRT_ENABLE_MEM_PIN", - "FI_OFI_RXM_USE_SRX", "D_LOG_FLUSH", "CRT_MRC_ENABLE", - "CRT_SECONDARY_PROVIDER", "D_PROVIDER_AUTH_KEY", "D_PORT_AUTO_ADJUST", - "D_POLL_TIMEOUT"}; + char *envars[] = {"D_PROVIDER", + "D_INTERFACE", + "D_DOMAIN", + "D_PORT", + "CRT_PHY_ADDR_STR", + "D_LOG_STDERR_IN_LOG", + "D_LOG_SIZE", + "D_LOG_FILE", + "D_LOG_FILE_APPEND_PID", + "D_LOG_MASK", + "DD_MASK", + "DD_STDERR", + "DD_SUBSYS", + "CRT_TIMEOUT", + "CRT_ATTACH_INFO_PATH", + "OFI_PORT", + "OFI_INTERFACE", + "OFI_DOMAIN", + "CRT_CREDIT_EP_CTX", + "CRT_CTX_SHARE_ADDR", + "CRT_CTX_NUM", + "D_FI_CONFIG", + "FI_UNIVERSE_SIZE", + "CRT_ENABLE_MEM_PIN", + "FI_OFI_RXM_USE_SRX", + "D_LOG_FLUSH", + "CRT_MRC_ENABLE", + "CRT_SECONDARY_PROVIDER", + "D_PROVIDER_AUTH_KEY", + "D_PORT_AUTO_ADJUST", + "D_POLL_TIMEOUT", + "D_POST_INIT", + "D_POST_INCR"}; D_INFO("-- ENVARS: --\n"); for (i = 0; i < ARRAY_SIZE(envars); i++) { @@ -237,12 +261,13 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, /* first step init - for initializing crt_gdata */ static int data_init(int server, crt_init_options_t *opt) { - uint32_t timeout; + uint32_t timeout = 0; uint32_t credits; uint32_t fi_univ_size = 0; uint32_t mem_pin_enable = 0; uint32_t is_secondary; char ucx_ib_fork_init = 0; + uint32_t post_init = CRT_HG_POST_INIT, post_incr = CRT_HG_POST_INCR; int rc = 0; D_DEBUG(DB_ALL, "initializing crt_gdata...\n"); @@ -252,6 +277,12 @@ static int data_init(int server, crt_init_options_t *opt) D_DEBUG(DB_ALL, "Starting RPCID %#lx. Num cores: %ld\n", crt_gdata.cg_rpcid, crt_gdata.cg_num_cores); + /* Set context post init / post incr to tune number of pre-posted recvs */ + d_getenv_int("D_POST_INIT", &post_init); + crt_gdata.cg_post_init = post_init; + d_getenv_int("D_POST_INCR", &post_incr); + crt_gdata.cg_post_incr = post_incr; + is_secondary = 0; /* Apply CART-890 workaround for server side only */ if (server) { @@ -264,13 +295,9 @@ static int data_init(int server, crt_init_options_t *opt) * is running using a secondary provider */ d_getenv_int("CRT_SECONDARY_PROVIDER", &is_secondary); - } - crt_gdata.cg_provider_is_primary = (is_secondary) ? 0 : 1; - timeout = 0; - if (opt && opt->cio_crt_timeout != 0) timeout = opt->cio_crt_timeout; else @@ -818,6 +845,7 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) crt_self_test_init(); + crt_iv_init(opt); rc = crt_opc_map_create(); if (rc != 0) { D_ERROR("crt_opc_map_create() failed, "DF_RC"\n", DP_RC(rc)); diff --git a/src/cart/crt_internal.h b/src/cart/crt_internal.h index 1067128249c..64392ee1c00 100644 --- a/src/cart/crt_internal.h +++ b/src/cart/crt_internal.h @@ -91,4 +91,6 @@ crt_hdlr_ctl_get_hostname(crt_rpc_t *rpc_req); void crt_hdlr_ctl_get_pid(crt_rpc_t *rpc_req); +void +crt_iv_init(crt_init_options_t *ops); #endif /* __CRT_INTERNAL_H__ */ diff --git a/src/cart/crt_internal_types.h b/src/cart/crt_internal_types.h index 18148ef1bd6..a09bc2b4c49 100644 --- a/src/cart/crt_internal_types.h +++ b/src/cart/crt_internal_types.h @@ -91,6 +91,10 @@ struct crt_gdata { /** */ struct crt_prov_gdata *cg_prov_gdata_secondary; + /** Hints to mercury for request post init (ignored for clients) */ + uint32_t cg_post_init; + uint32_t cg_post_incr; + /** global timeout value (second) for all RPCs */ uint32_t cg_timeout; @@ -100,6 +104,7 @@ struct crt_gdata { /** credits limitation for #inflight RPCs per target EP CTX */ uint32_t cg_credit_ep_ctx; + uint32_t cg_iv_inline_limit; /** the global opcode map */ struct crt_opc_map *cg_opc_map; /** HG level global data */ diff --git a/src/cart/crt_iv.c b/src/cart/crt_iv.c index 3e65025733a..801959eeee2 100644 --- a/src/cart/crt_iv.c +++ b/src/cart/crt_iv.c @@ -135,6 +135,14 @@ struct crt_ivns_internal { void *cii_user_priv; }; +void +crt_iv_init(crt_init_options_t *ops) +{ + if (ops != NULL && ops->cio_max_unexpected_size > 1024) + crt_gdata.cg_iv_inline_limit = ops->cio_max_expected_size - 1024; + D_INFO("max inline buf size is %u\n", crt_gdata.cg_iv_inline_limit); +} + static void handle_response_cb(const struct crt_cb_info *cb_info); @@ -381,14 +389,13 @@ crt_ivf_finalize(struct iv_fetch_cb_info *iv_info, crt_iv_key_t *iv_key, if (rpc) { /* If there is child to respond to - bulk transfer to it */ - if (output_rc == 0) { + if (output_rc == 0 && iv_info->ifc_child_bulk != CRT_BULK_NULL) { /* Note: function will increment ref count on 'rpc' */ rc = crt_ivf_bulk_transfer(iv_info->ifc_ivns_internal, iv_info->ifc_class_id, iv_key, iv_value, iv_info->ifc_child_bulk, - rpc, - iv_info->ifc_user_priv); + rpc, iv_info->ifc_user_priv); if (rc != 0) D_ERROR("Bulk transfer failed for key=%p\n", iv_key); @@ -399,7 +406,7 @@ crt_ivf_finalize(struct iv_fetch_cb_info *iv_info, crt_iv_key_t *iv_key, output = crt_reply_get(rpc); output->ifo_rc = output_rc; - + output->ifo_sgl = *iv_value; /* Reply can fail */ crt_reply_send(rpc); } @@ -434,7 +441,6 @@ crt_ivf_pending_reqs_process(struct crt_ivns_internal *ivns_internal, struct iv_fetch_cb_info *iv_info; struct crt_iv_fetch_out *output; int rc = 0; - bool put_needed = false; iv_ops = crt_iv_ops_get(ivns_internal, class_id); D_ASSERT(iv_ops != NULL); @@ -450,7 +456,8 @@ crt_ivf_pending_reqs_process(struct crt_ivns_internal *ivns_internal, &kip_entry->kip_pending_fetch_list, struct pending_fetch, pf_link))) { - d_sg_list_t tmp_iv_value = {0}; + d_sg_list_t tmp_iv_value = {0}; + bool put_needed = false; iv_info = pending_fetch->pf_cb_info; @@ -489,8 +496,6 @@ crt_ivf_pending_reqs_process(struct crt_ivns_internal *ivns_internal, CRT_IV_PERM_READ, &tmp_iv_value, &iv_info->ifc_user_priv); - - put_needed = false; if (rc == 0) { put_needed = true; rc = iv_ops->ivo_on_fetch(ivns_internal, @@ -501,13 +506,26 @@ crt_ivf_pending_reqs_process(struct crt_ivns_internal *ivns_internal, if (rc == 0) { /* Function will do IVNS_ADDREF if needed */ - rc = crt_ivf_bulk_transfer(ivns_internal, - class_id, - &iv_info->ifc_iv_key, - &tmp_iv_value, - iv_info->ifc_child_bulk, - iv_info->ifc_child_rpc, - iv_info->ifc_user_priv); + if (iv_info->ifc_child_bulk != CRT_BULK_NULL) { + rc = crt_ivf_bulk_transfer(ivns_internal, class_id, + &iv_info->ifc_iv_key, + &tmp_iv_value, + iv_info->ifc_child_bulk, + iv_info->ifc_child_rpc, + iv_info->ifc_user_priv); + /* bulk transfer callback will put if succeed */ + if (rc == 0) + put_needed = false; + } else { + /* inline reply */ + output = crt_reply_get(iv_info->ifc_child_rpc); + output->ifo_rc = 0; + output->ifo_sgl = tmp_iv_value; + rc = crt_reply_send(iv_info->ifc_child_rpc); + if (rc != 0) + D_ERROR("crt_reply_send(): "DF_RC"\n", + DP_RC(rc)); + } } else { D_ERROR("Failed to process pending request\n"); @@ -517,7 +535,7 @@ crt_ivf_pending_reqs_process(struct crt_ivns_internal *ivns_internal, crt_reply_send(iv_info->ifc_child_rpc); } - if (rc != 0 && put_needed) + if (put_needed) iv_ops->ivo_on_put(ivns_internal, &tmp_iv_value, iv_info->ifc_user_priv); @@ -547,12 +565,8 @@ crt_ivf_pending_reqs_process(struct crt_ivns_internal *ivns_internal, 0, CRT_IV_PERM_READ, &tmp_iv_value, &iv_info->ifc_user_priv); - - put_needed = false; - if (rc == 0) { put_needed = true; - rc = iv_ops->ivo_on_fetch(ivns_internal, &iv_info->ifc_iv_key, 0x0, @@ -940,11 +954,8 @@ crt_ivf_bulk_transfer_done_cb(const struct crt_bulk_cb_info *info) cb_info->tci_class_id); D_ASSERT(iv_ops != NULL); - rc = iv_ops->ivo_on_put(cb_info->tci_ivns_internal, - &cb_info->tci_iv_value, - cb_info->tci_user_priv); - if (rc != 0) - D_ERROR("ivo_on_put(): "DF_RC"\n", DP_RC(rc)); + iv_ops->ivo_on_put(cb_info->tci_ivns_internal, &cb_info->tci_iv_value, + cb_info->tci_user_priv); rc = crt_reply_send(rpc); if (rc != 0) @@ -971,8 +982,7 @@ crt_ivf_bulk_transfer(struct crt_ivns_internal *ivns_internal, crt_bulk_opid_t opid; crt_bulk_t bulk_hdl; struct crt_iv_fetch_out *output; - int size; - int i; + size_t size; int rc2; int rc = 0; @@ -990,10 +1000,7 @@ crt_ivf_bulk_transfer(struct crt_ivns_internal *ivns_internal, } /* Calculate total size of all iovs in sg list */ - size = 0; - for (i = 0; i < iv_value->sg_nr; i++) - size += iv_value->sg_iovs[i].iov_buf_len; - + size = d_sgl_buf_size(iv_value); /* crt_req_decref done in crt_ivf_bulk_transfer_done_cb */ RPC_PUB_ADDREF(rpc); @@ -1053,6 +1060,7 @@ handle_ivfetch_response(const struct crt_cb_info *cb_info) struct crt_iv_ops *iv_ops; struct crt_ivns_internal *ivns; struct ivf_key_in_progress *kip_entry; + d_sg_list_t *iv_value = NULL; uint32_t class_id; int rc; @@ -1069,11 +1077,17 @@ handle_ivfetch_response(const struct crt_cb_info *cb_info) IV_DBG(&input->ifi_key, "response received, rc = %d\n", rc); + if (rc == 0) { + /* Inline IV fetch */ + if (iv_info->ifc_bulk_hdl == NULL) + d_sgl_buf_copy(&iv_info->ifc_iv_value, &output->ifo_sgl); + + iv_value = &iv_info->ifc_iv_value; + } + /* In case of a failure, call on_refresh with NULL iv_value */ - iv_ops->ivo_on_refresh(ivns, &input->ifi_key, - 0, /* TODO: iv_ver */ - rc == 0 ? &iv_info->ifc_iv_value : NULL, - false, rc, iv_info->ifc_user_priv); + iv_ops->ivo_on_refresh(ivns, &input->ifi_key, 0, /* TODO: iv_ver */ + iv_value, false, rc, iv_info->ifc_user_priv); if (iv_info->ifc_bulk_hdl) crt_bulk_free(iv_info->ifc_bulk_hdl); @@ -1171,11 +1185,14 @@ crt_ivf_rpc_issue(d_rank_t dest_node, crt_iv_key_t *iv_key, D_MUTEX_UNLOCK(&entry->kip_lock); D_MUTEX_UNLOCK(&ivns_internal->cii_lock); - rc = crt_bulk_create(ivns_internal->cii_ctx, iv_value, CRT_BULK_RW, - &local_bulk); - if (rc != 0) { - D_ERROR("crt_bulk_create(): "DF_RC"\n", DP_RC(rc)); - D_GOTO(exit, rc); + if (d_sgl_buf_size(iv_value) > crt_gdata.cg_iv_inline_limit) { + IV_DBG(iv_key, "bulk transfer with size %zu\n", d_sgl_buf_size(iv_value)); + rc = crt_bulk_create(ivns_internal->cii_ctx, iv_value, CRT_BULK_RW, + &local_bulk); + if (rc != 0) { + D_ERROR("crt_bulk_create(): "DF_RC"\n", DP_RC(rc)); + D_GOTO(exit, rc); + } } /* Note: destination node is using global rank already */ @@ -1192,9 +1209,10 @@ crt_ivf_rpc_issue(d_rank_t dest_node, crt_iv_key_t *iv_key, input = crt_req_get(rpc); D_ASSERT(input != NULL); - input->ifi_value_bulk = local_bulk; - - cb_info->ifc_bulk_hdl = local_bulk; + if (local_bulk != CRT_BULK_NULL) { + input->ifi_value_bulk = local_bulk; + cb_info->ifc_bulk_hdl = local_bulk; + } d_iov_set(&input->ifi_key, iv_key->iov_buf, iv_key->iov_buf_len); input->ifi_class_id = cb_info->ifc_class_id; @@ -1310,12 +1328,11 @@ crt_hdlr_iv_fetch_aux(void *arg) d_sg_list_t iv_value = {0}; bool put_needed = false; void *user_priv = NULL; - crt_rpc_t *rpc_req; + crt_rpc_t *rpc_req = arg; uint32_t grp_ver_entry; uint32_t grp_ver_current; int rc = 0; - rpc_req = arg; input = crt_req_get(rpc_req); output = crt_reply_get(rpc_req); @@ -1326,7 +1343,7 @@ crt_hdlr_iv_fetch_aux(void *arg) ivns_internal = crt_ivns_internal_lookup(&ivns_id); if (ivns_internal == NULL) { D_ERROR("Failed to lookup ivns internal!\n"); - D_GOTO(send_error, rc = -DER_NONEXIST); + D_GOTO(reply_direct, rc = -DER_NONEXIST); } /* This function is called with ivns_internal ref count held. Since @@ -1349,13 +1366,13 @@ crt_hdlr_iv_fetch_aux(void *arg) "Group (%s) version mismatch. Local: %d Remote :%d\n", ivns_id.ii_group_name, grp_ver_entry, input->ifi_grp_ver); - D_GOTO(send_error, rc = -DER_GRPVER); + D_GOTO(reply_direct, rc = -DER_GRPVER); } iv_ops = crt_iv_ops_get(ivns_internal, input->ifi_class_id); if (iv_ops == NULL) { D_ERROR("Returned iv_ops were NULL\n"); - D_GOTO(send_error, rc = -DER_INVAL); + D_GOTO(reply_direct, rc = -DER_INVAL); } IV_DBG(&input->ifi_key, "fetch handler entered\n"); @@ -1363,23 +1380,28 @@ crt_hdlr_iv_fetch_aux(void *arg) 0, CRT_IV_PERM_READ, &iv_value, &user_priv); if (rc != 0) { D_ERROR("ivo_on_get(): "DF_RC"\n", DP_RC(rc)); - D_GOTO(send_error, rc); + D_GOTO(reply_direct, rc); } put_needed = true; - rc = iv_ops->ivo_on_fetch(ivns_internal, &input->ifi_key, 0, 0x0, &iv_value, user_priv); if (rc == 0) { /* Note: This increments ref count on 'rpc_req' and ivns */ - rc = crt_ivf_bulk_transfer(ivns_internal, - input->ifi_class_id, - &input->ifi_key, - &iv_value, input->ifi_value_bulk, - rpc_req, user_priv); - if (rc != 0) { - D_ERROR("bulk transfer failed; "DF_RC"\n", DP_RC(rc)); - D_GOTO(send_error, rc); + if (input->ifi_value_bulk != CRT_BULK_NULL) { + rc = crt_ivf_bulk_transfer(ivns_internal, + input->ifi_class_id, + &input->ifi_key, + &iv_value, input->ifi_value_bulk, + rpc_req, user_priv); + if (rc != 0) { + DL_ERROR(rc, "bulk transfer failed"); + D_GOTO(reply_direct, rc); + } + put_needed = false; + } else { + output->ifo_sgl = iv_value; + D_GOTO(reply_direct, rc); } } else if (rc == -DER_IVCB_FORWARD) { /* Forward the request to the parent */ @@ -1389,15 +1411,11 @@ crt_hdlr_iv_fetch_aux(void *arg) if (ivns_internal->cii_grp_priv->gp_self == input->ifi_root_node) { D_ERROR("Forward requested for root node\n"); - D_GOTO(send_error, rc = -DER_INVAL); + D_GOTO(reply_direct, rc = -DER_INVAL); } - rc = iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); + iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); put_needed = false; - if (rc != 0) { - D_ERROR("ivo_on_put(): "DF_RC"\n", DP_RC(rc)); - D_GOTO(send_error, rc); - } /* Reset the iv_value, since it maybe freed in on_put() */ memset(&iv_value, 0, sizeof(iv_value)); @@ -1406,7 +1424,7 @@ crt_hdlr_iv_fetch_aux(void *arg) &user_priv); if (rc != 0) { D_ERROR("ivo_on_get(): "DF_RC"\n", DP_RC(rc)); - D_GOTO(send_error, rc); + D_GOTO(reply_direct, rc); } put_needed = true; @@ -1420,7 +1438,7 @@ crt_hdlr_iv_fetch_aux(void *arg) if (rc != 0) { D_DEBUG(DB_TRACE, "crt_iv_parent_get() returned %d\n", rc); - D_GOTO(send_error, rc = -DER_OOG); + D_GOTO(reply_direct, rc = -DER_OOG); } /* Check here for change in group */ @@ -1429,12 +1447,12 @@ crt_hdlr_iv_fetch_aux(void *arg) "On Entry: %d:: Changed To :%d\n", ivns_id.ii_group_name, grp_ver_entry, grp_ver_current); - D_GOTO(send_error, rc = -DER_GRPVER); + D_GOTO(reply_direct, rc = -DER_GRPVER); } D_ALLOC_PTR(cb_info); if (cb_info == NULL) - D_GOTO(send_error, rc = -DER_NOMEM); + D_GOTO(reply_direct, rc = -DER_NOMEM); cb_info->ifc_child_rpc = rpc_req; cb_info->ifc_child_bulk = input->ifi_value_bulk; @@ -1462,11 +1480,11 @@ crt_hdlr_iv_fetch_aux(void *arg) IVNS_DECREF(cb_info->ifc_ivns_internal); D_FREE(cb_info); - D_GOTO(send_error, rc); + D_GOTO(reply_direct, rc); } } else { D_ERROR("ERROR happened: "DF_RC"\n", DP_RC(rc)); - D_GOTO(send_error, rc); + D_GOTO(reply_direct, rc); } /* addref in crt_hdlr_iv_fetch */ @@ -1477,16 +1495,14 @@ crt_hdlr_iv_fetch_aux(void *arg) IVNS_DECREF(ivns_internal); return; -send_error: - if (put_needed && iv_ops) - iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); +reply_direct: output->ifo_rc = rc; rc = crt_reply_send(rpc_req); - if (rc != DER_SUCCESS) { + if (rc != DER_SUCCESS) D_ERROR("crt_reply_send(opc: %#x): "DF_RC"\n", rpc_req->cr_opc, DP_RC(rc)); - } - + if (put_needed && iv_ops) + iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); /* ADDREF done in lookup above */ if (ivns_internal) IVNS_DECREF(ivns_internal); @@ -1798,7 +1814,6 @@ crt_hdlr_iv_sync_aux(void *arg) struct crt_iv_ops *iv_ops = NULL; struct crt_ivns_id ivns_id; crt_iv_sync_t *sync_type; - d_sg_list_t iv_value = {0}; bool need_put = false; void *user_priv = NULL; crt_rpc_t *rpc_req; @@ -1849,7 +1864,7 @@ crt_hdlr_iv_sync_aux(void *arg) D_ASSERT(iv_ops != NULL); /* If bulk is not set, we issue invalidate call */ - if (rpc_req->cr_co_bulk_hdl == CRT_BULK_NULL) { + if (rpc_req->cr_co_bulk_hdl == CRT_BULK_NULL && input->ivs_sync_sgl.sg_nr == 0) { rc = iv_ops->ivo_on_refresh(ivns_internal, &input->ivs_key, 0, NULL, true, 0x0, NULL); D_GOTO(exit, rc); @@ -1860,10 +1875,10 @@ crt_hdlr_iv_sync_aux(void *arg) case CRT_IV_SYNC_EVENT_UPDATE: { d_sg_list_t tmp_iv; - d_iov_t *tmp_iovs; + d_iov_t tmp_iovs; rc = iv_ops->ivo_on_get(ivns_internal, &input->ivs_key, - 0, CRT_IV_PERM_READ, &iv_value, + 0, CRT_IV_PERM_READ | CRT_IV_NO_ALLOC, &tmp_iv, &user_priv); if (rc != 0) { D_ERROR("ivo_on_get(): "DF_RC"\n", DP_RC(rc)); @@ -1872,36 +1887,27 @@ crt_hdlr_iv_sync_aux(void *arg) need_put = true; - D_ALLOC_ARRAY(tmp_iovs, iv_value.sg_nr); - if (tmp_iovs == NULL) { - D_GOTO(exit, rc = -DER_NOMEM); - } - - tmp_iv.sg_nr = iv_value.sg_nr; - tmp_iv.sg_iovs = tmp_iovs; - - /* Populate tmp_iv.sg_iovs[0] to [sg_nr] */ - rc = crt_bulk_access(rpc_req->cr_co_bulk_hdl, &tmp_iv); - if (rc != 0) { - D_FREE(tmp_iovs); - D_ERROR("crt_bulk_access(): "DF_RC"\n", DP_RC(rc)); - D_GOTO(exit, rc); + if (rpc_req->cr_co_bulk_hdl != CRT_BULK_NULL) { + tmp_iv.sg_iovs = &tmp_iovs; + /* Populate tmp_iv.sg_iovs[0] to [sg_nr] */ + rc = crt_bulk_access(rpc_req->cr_co_bulk_hdl, &tmp_iv); + if (rc != 0) { + D_ERROR("crt_bulk_access(): "DF_RC"\n", DP_RC(rc)); + D_GOTO(exit, rc); + } + } else if (input->ivs_sync_sgl.sg_nr > 0) { + tmp_iv = input->ivs_sync_sgl; } rc = iv_ops->ivo_on_refresh(ivns_internal, &input->ivs_key, 0, &tmp_iv, false, 0, user_priv); - D_FREE(tmp_iovs); if (rc != 0) { D_ERROR("ivo_on_refresh(): "DF_RC"\n", DP_RC(rc)); D_GOTO(exit, rc); } - rc = iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); + iv_ops->ivo_on_put(ivns_internal, NULL, user_priv); need_put = false; - if (rc != 0) { - D_ERROR("ivo_on_put(): "DF_RC"\n", DP_RC(rc)); - D_GOTO(exit, rc); - } break; } @@ -1924,7 +1930,7 @@ crt_hdlr_iv_sync_aux(void *arg) exit: if (need_put && iv_ops) - iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); + iv_ops->ivo_on_put(ivns_internal, NULL, user_priv); output->rc = rc; crt_reply_send(rpc_req); @@ -2072,6 +2078,8 @@ call_pre_sync_cb(struct crt_ivns_internal *ivns_internal, D_ERROR("crt_bulk_access(): "DF_RC"\n", DP_RC(rc)); D_GOTO(exit, rc); } + } else if (input->ivs_sync_sgl.sg_nr > 0) { + tmp_iv = input->ivs_sync_sgl; } D_DEBUG(DB_TRACE, "Executing ivo_pre_sync\n"); @@ -2264,8 +2272,9 @@ crt_ivsync_rpc_issue(struct crt_ivns_internal *ivns_internal, uint32_t class_id, } local_bulk = CRT_BULK_NULL; - if (iv_value != NULL) { - D_DEBUG(DB_TRACE, "Create Bulk\n"); + if (iv_value != NULL && d_sgl_buf_size(iv_value) > crt_gdata.cg_iv_inline_limit) { + IV_DBG(iv_key, "bulk transfer with size %zu\n", + d_sgl_buf_size(iv_value)); rc = crt_bulk_create(ivns_internal->cii_ctx, iv_value, CRT_BULK_RO, &local_bulk); if (rc != 0) { @@ -2290,10 +2299,8 @@ crt_ivsync_rpc_issue(struct crt_ivns_internal *ivns_internal, uint32_t class_id, D_ASSERT(input != NULL); D_ALLOC_PTR(iv_sync_cb); - if (iv_sync_cb == NULL) { - /* Avoid checkpatch warning */ + if (iv_sync_cb == NULL) D_GOTO(exit, rc = -DER_NOMEM); - } iv_sync_cb->isc_sync_type = *sync_type; input->ivs_ivns_id = ivns_internal->cii_gns.gn_ivns_id.ii_nsid; @@ -2305,6 +2312,10 @@ crt_ivsync_rpc_issue(struct crt_ivns_internal *ivns_internal, uint32_t class_id, d_iov_set(&input->ivs_sync_type, &iv_sync_cb->isc_sync_type, sizeof(crt_iv_sync_t)); + /* Set inline IV buffer if needed */ + if (iv_value != NULL && local_bulk == CRT_BULK_NULL) + input->ivs_sync_sgl = *iv_value; + input->ivs_class_id = class_id; iv_sync_cb->isc_bulk_hdl = local_bulk; @@ -2414,7 +2425,8 @@ finalize_transfer_back(struct update_cb_info *cb_info, int rc) crt_reply_send(cb_info->uci_child_rpc); /* ADDREF done in crt_hdlr_iv_update */ - crt_bulk_free(cb_info->uci_bulk_hdl); + if (cb_info->uci_bulk_hdl != CRT_BULK_NULL) + crt_bulk_free(cb_info->uci_bulk_hdl); iv_ops = crt_iv_ops_get(ivns, cb_info->uci_class_id); D_ASSERT(iv_ops != NULL); @@ -2439,16 +2451,16 @@ bulk_update_transfer_back_done(const struct crt_bulk_cb_info *info) } /* Helper function to transfer iv_value back to child */ -static -int transfer_back_to_child(crt_iv_key_t *key, struct update_cb_info *cb_info, - bool do_refresh, int update_rc) +static int +transfer_back_to_child(crt_iv_key_t *key, struct update_cb_info *cb_info, + int update_rc) { struct crt_bulk_desc bulk_desc = {0}; struct crt_iv_update_in *child_input; + struct crt_iv_update_out *child_output; struct crt_ivns_internal *ivns; struct crt_iv_ops *iv_ops; - int size = 0; - int i; + size_t size = 0; int rc = 0; ivns = cb_info->uci_ivns_internal; @@ -2456,11 +2468,6 @@ int transfer_back_to_child(crt_iv_key_t *key, struct update_cb_info *cb_info, iv_ops = crt_iv_ops_get(ivns, cb_info->uci_class_id); D_ASSERT(iv_ops != NULL); - if (do_refresh) - iv_ops->ivo_on_refresh(ivns, key, 0, - &cb_info->uci_iv_value, - false, update_rc, cb_info->uci_user_priv); - /* No more children -- we are the originator; call update_cb */ if (cb_info->uci_child_rpc == NULL) { cb_info->uci_comp_cb(ivns, cb_info->uci_class_id, key, NULL, @@ -2479,13 +2486,18 @@ int transfer_back_to_child(crt_iv_key_t *key, struct update_cb_info *cb_info, return 0; } - /* Perform bulk transfer back to the child */ child_input = crt_req_get(cb_info->uci_child_rpc); + if (child_input->ivu_iv_value_bulk == CRT_BULK_NULL) { + child_output = crt_reply_get(cb_info->uci_child_rpc); + D_ASSERT(update_rc != 0 || child_output->ivo_iv_sgl.sg_nr > 0); + finalize_transfer_back(cb_info, update_rc); + return 0; + } - /* Calculate size of iv value */ - for (i = 0; i < cb_info->uci_iv_value.sg_nr; i++) - size += cb_info->uci_iv_value.sg_iovs[i].iov_buf_len; + /* Perform bulk transfer back to the child */ + /* Calculate size of iv value */ + size = d_sgl_buf_size(&cb_info->uci_iv_value); bulk_desc.bd_rpc = cb_info->uci_child_rpc; bulk_desc.bd_bulk_op = CRT_BULK_PUT; bulk_desc.bd_remote_hdl = child_input->ivu_iv_value_bulk; @@ -2515,36 +2527,35 @@ handle_ivupdate_response(const struct crt_cb_info *cb_info) struct crt_iv_ops *iv_ops; int rc; + iv_ops = crt_iv_ops_get(iv_info->uci_ivns_internal, iv_info->uci_class_id); + D_ASSERT(iv_ops != NULL); + /* For bi-directional updates, transfer data back to child */ if (iv_info->uci_sync_type.ivs_flags & CRT_IV_SYNC_BIDIRECTIONAL) { - transfer_back_to_child(&input->ivu_key, iv_info, true, - cb_info->cci_rc ?: output->rc); + if (iv_info->uci_bulk_hdl == CRT_BULK_NULL) + d_sgl_buf_copy(&iv_info->uci_iv_value, &output->ivo_iv_sgl); + iv_ops->ivo_on_refresh(iv_info->uci_ivns_internal, &input->ivu_key, 0, + &iv_info->uci_iv_value, false, cb_info->cci_rc ?: output->rc, + iv_info->uci_user_priv); + + if (input->ivu_iv_value_bulk == CRT_BULK_NULL && + iv_info->uci_child_rpc != NULL) { + child_output = crt_reply_get(iv_info->uci_child_rpc); + child_output->ivo_iv_sgl = iv_info->uci_iv_value; + } + transfer_back_to_child(&input->ivu_key, iv_info, cb_info->cci_rc ?: output->rc); D_GOTO(exit, 0); } - iv_ops = crt_iv_ops_get(iv_info->uci_ivns_internal, - iv_info->uci_class_id); - D_ASSERT(iv_ops != NULL); - if (iv_info->uci_child_rpc) { child_output = crt_reply_get(iv_info->uci_child_rpc); /* uci_bulk_hdl will not be set for invalidate call */ - if (iv_info->uci_bulk_hdl != CRT_BULK_NULL) { + if (iv_info->uci_bulk_hdl != CRT_BULK_NULL) crt_bulk_free(iv_info->uci_bulk_hdl); - rc = iv_ops->ivo_on_put(iv_info->uci_ivns_internal, - &iv_info->uci_iv_value, - iv_info->uci_user_priv); - - if (rc != 0) { - D_ERROR("ivo_on_put(): "DF_RC"\n", DP_RC(rc)); - child_output->rc = rc; - } else { - child_output->rc = output->rc; - } - } else { - child_output->rc = output->rc; - } + iv_ops->ivo_on_put(iv_info->uci_ivns_internal, &iv_info->uci_iv_value, + iv_info->uci_user_priv); + child_output->rc = output->rc; if (cb_info->cci_rc != 0) child_output->rc = cb_info->cci_rc; @@ -2559,7 +2570,7 @@ handle_ivupdate_response(const struct crt_cb_info *cb_info) } else { d_sg_list_t *tmp_iv_value; - if (iv_info->uci_bulk_hdl == CRT_BULK_NULL) + if (iv_info->uci_iv_value.sg_nr == 0) tmp_iv_value = NULL; else tmp_iv_value = &iv_info->uci_iv_value; @@ -2583,9 +2594,8 @@ handle_ivupdate_response(const struct crt_cb_info *cb_info) if (iv_info->uci_bulk_hdl != CRT_BULK_NULL) crt_bulk_free(iv_info->uci_bulk_hdl); if (rc != 0) { - rc = iv_ops->ivo_on_put(iv_info->uci_ivns_internal, - tmp_iv_value, - iv_info->uci_user_priv); + iv_ops->ivo_on_put(iv_info->uci_ivns_internal, tmp_iv_value, + iv_info->uci_user_priv); } } @@ -2628,9 +2638,14 @@ crt_ivu_rpc_issue(d_rank_t dest_rank, crt_iv_key_t *iv_key, /* Update with NULL value is invalidate call */ if (iv_value) { - rc = crt_bulk_create(ivns_internal->cii_ctx, iv_value, - CRT_BULK_RW, &local_bulk); - + if (d_sgl_buf_size(iv_value) > crt_gdata.cg_iv_inline_limit) { + IV_DBG(iv_key, "bulk transfer with size %zu\n", + d_sgl_buf_size(iv_value)); + rc = crt_bulk_create(ivns_internal->cii_ctx, iv_value, + CRT_BULK_RW, &local_bulk); + } else { + input->ivu_iv_sgl = *iv_value; + } if (rc != 0) { D_ERROR("crt_bulk_create(): "DF_RC"\n", DP_RC(rc)); D_GOTO(exit, rc); @@ -2881,18 +2896,16 @@ bulk_update_transfer_done_aux(const struct crt_bulk_cb_info *info) } else if (update_rc == 0) { /* If sync was bi-directional - transfer value back */ if (sync_type->ivs_flags & CRT_IV_SYNC_BIDIRECTIONAL) { - rc = transfer_back_to_child(&input->ivu_key, - update_cb_info, - false, update_rc); + rc = transfer_back_to_child(&input->ivu_key, update_cb_info, + update_rc); if (rc == 0) rc = update_rc; D_GOTO(exit, rc); } crt_bulk_free(cb_info->buc_bulk_hdl); - output->rc = iv_ops->ivo_on_put(ivns_internal, - &cb_info->buc_iv_value, - cb_info->buc_user_priv); + output->rc = -DER_SUCCESS; + iv_ops->ivo_on_put(ivns_internal, &cb_info->buc_iv_value, cb_info->buc_user_priv); crt_reply_send(info->bci_bulk_desc->bd_rpc); RPC_PUB_DECREF(info->bci_bulk_desc->bd_rpc); @@ -3018,6 +3031,120 @@ bulk_update_transfer_done(const struct crt_bulk_cb_info *info) return rc; } +static void +crt_iv_update_inline_hdlr(crt_rpc_t *rpc, struct crt_ivns_internal *ivns_internal) +{ + struct crt_iv_ops *iv_ops; + struct crt_iv_update_in *input; + struct crt_iv_update_out *output; + d_sg_list_t iv_value = {0}; + struct update_cb_info *update_cb_info = NULL; + int rc = 0; + d_rank_t next_rank; + crt_iv_sync_t *sync_type; + void *user_priv; + uint32_t grp_ver; + + input = crt_req_get(rpc); + output = crt_reply_get(rpc); + + D_ASSERT(input != NULL); + D_ASSERT(output != NULL); + + iv_ops = crt_iv_ops_get(ivns_internal, input->ivu_class_id); + D_ASSERT(iv_ops != NULL); + + rc = iv_ops->ivo_on_get(ivns_internal, &input->ivu_key, 0, + CRT_IV_PERM_WRITE, &iv_value, &user_priv); + if (rc != 0) { + D_ERROR("ivo_on_get(): "DF_RC"\n", DP_RC(rc)); + D_GOTO(send_error, rc); + } + + d_sgl_buf_copy(&iv_value, &input->ivu_iv_sgl); + + rc = iv_ops->ivo_on_update(ivns_internal, &input->ivu_key, 0, false, + &iv_value, user_priv); + + sync_type = input->ivu_sync_type.iov_buf; + + if (rc == -DER_IVCB_FORWARD || + (rc == 0 && sync_type->ivs_flags & CRT_IV_SYNC_BIDIRECTIONAL)) { + + D_ALLOC_PTR(update_cb_info); + if (update_cb_info == NULL) + D_GOTO(put_error, rc = -DER_NOMEM); + + update_cb_info->uci_child_rpc = rpc; + + update_cb_info->uci_ivns_internal = ivns_internal; + IVNS_ADDREF(ivns_internal); + + update_cb_info->uci_class_id = input->ivu_class_id; + update_cb_info->uci_caller_rank = input->ivu_caller_node; + update_cb_info->uci_sync_type = *sync_type; + update_cb_info->uci_user_priv = user_priv; + update_cb_info->uci_iv_value = iv_value; + if (rc == -DER_IVCB_FORWARD) { + /* + * Forward request to the parent + * Get group version to associate with next_rank. + * Pass it down to crt_ivu_rpc_issue + */ + D_RWLOCK_RDLOCK(&ivns_internal->cii_grp_priv->gp_rwlock); + grp_ver = ivns_internal->cii_grp_priv->gp_membs_ver; + + rc = crt_iv_parent_get(ivns_internal, input->ivu_root_node, &next_rank); + D_RWLOCK_UNLOCK(&ivns_internal->cii_grp_priv->gp_rwlock); + + if (rc != 0) { + D_DEBUG(DB_TRACE, "crt_iv_parent_get() returned %d\n", rc); + D_GOTO(decref_error, rc = -DER_OOG); + } + + rc = crt_ivu_rpc_issue(next_rank, &input->ivu_key, + &iv_value, sync_type, + input->ivu_root_node, grp_ver, + update_cb_info); + if (rc != 0) { + D_ERROR("crt_ivu_rpc_issue(): "DF_RC"\n", DP_RC(rc)); + D_GOTO(decref_error, rc); + } + RPC_PUB_ADDREF(update_cb_info->uci_child_rpc); + } else { + int rc1; + + /* If sync was bi-directional - transfer value back */ + output->ivo_iv_sgl = iv_value; + RPC_PUB_ADDREF(update_cb_info->uci_child_rpc); + rc1 = transfer_back_to_child(&input->ivu_key, update_cb_info, rc); + if (rc == 0) + rc = rc1; + + D_GOTO(exit, rc); + } + } else if (rc == 0) { + output->rc = rc; + crt_reply_send(rpc); + iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); + } else { + D_GOTO(put_error, rc); + } +exit: + return; + +decref_error: + IVNS_DECREF(ivns_internal); +put_error: + iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); + +send_error: + output->rc = rc; + crt_reply_send(rpc); + if (update_cb_info) + D_FREE(update_cb_info); +} + /* IV UPDATE RPC handler */ void crt_hdlr_iv_update(crt_rpc_t *rpc_req) @@ -3034,9 +3161,8 @@ crt_hdlr_iv_update(crt_rpc_t *rpc_req) crt_iv_sync_t *sync_type; d_rank_t next_rank; struct update_cb_info *update_cb_info; - int size; + size_t size; void *user_priv; - int i; uint32_t grp_ver_entry; uint32_t grp_ver_current; bool put_needed = false; @@ -3078,7 +3204,7 @@ crt_hdlr_iv_update(crt_rpc_t *rpc_req) D_GOTO(send_error, rc = -DER_INVAL); } - if (input->ivu_iv_value_bulk == CRT_BULK_NULL) { + if (input->ivu_iv_value_bulk == CRT_BULK_NULL && input->ivu_iv_sgl.sg_nr == 0) { rc = iv_ops->ivo_on_refresh(ivns_internal, &input->ivu_key, 0, NULL, true, 0, NULL); if (rc == -DER_IVCB_FORWARD) { @@ -3156,6 +3282,11 @@ crt_hdlr_iv_update(crt_rpc_t *rpc_req) D_GOTO(exit, rc = 0); } + if (input->ivu_iv_value_bulk == CRT_BULK_NULL) { + crt_iv_update_inline_hdlr(rpc_req, ivns_internal); + D_GOTO(exit, rc); + } + rc = iv_ops->ivo_on_get(ivns_internal, &input->ivu_key, 0, CRT_IV_PERM_WRITE, &iv_value, &user_priv); if (rc != 0) { @@ -3164,10 +3295,7 @@ crt_hdlr_iv_update(crt_rpc_t *rpc_req) } put_needed = true; - size = 0; - for (i = 0; i < iv_value.sg_nr; i++) - size += iv_value.sg_iovs[i].iov_buf_len; - + size = d_sgl_buf_size(&iv_value); rc = crt_bulk_create(rpc_req->cr_ctx, &iv_value, CRT_BULK_RW, &local_bulk_handle); if (rc != 0) { @@ -3219,7 +3347,7 @@ crt_hdlr_iv_update(crt_rpc_t *rpc_req) crt_reply_send(rpc_req); if (put_needed) - iv_ops->ivo_on_put(ivns_internal, &iv_value, &user_priv); + iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); /* ADDREF done in lookup above */ if (ivns_internal) diff --git a/src/cart/crt_rpc.h b/src/cart/crt_rpc.h index 6ddda306452..d297d2f13f0 100644 --- a/src/cart/crt_rpc.h +++ b/src/cart/crt_rpc.h @@ -220,7 +220,7 @@ crt_rpc_unlock(struct crt_rpc_priv *rpc_priv) #define CRT_PROTO_FI_VERSION 3 #define CRT_PROTO_ST_VERSION 1 #define CRT_PROTO_CTL_VERSION 1 -#define CRT_PROTO_IV_VERSION 1 +#define CRT_PROTO_IV_VERSION 2 /* LIST of internal RPCS in form of: * OPCODE, flags, FMT, handler, corpc_hdlr, @@ -308,7 +308,7 @@ crt_rpc_unlock(struct crt_rpc_priv *rpc_priv) crt_hdlr_iv_update, NULL) \ X(CRT_OPC_IV_SYNC, \ 0, &CQF_crt_iv_sync, \ - crt_hdlr_iv_sync, &crt_iv_sync_co_ops) \ + crt_hdlr_iv_sync, &crt_iv_sync_co_ops) /* Define for RPC enum population below */ #define X(a, b, c, d, e) a, @@ -476,7 +476,8 @@ CRT_RPC_DECLARE(crt_st_status_req, ((d_rank_t) (ifi_root_node) CRT_VAR) #define CRT_OSEQ_IV_FETCH /* output fields */ \ - ((int32_t) (ifo_rc) CRT_VAR) + ((d_sg_list_t) (ifo_sgl) CRT_VAR) \ + ((int32_t) (ifo_rc) CRT_VAR) \ CRT_RPC_DECLARE(crt_iv_fetch, CRT_ISEQ_IV_FETCH, CRT_OSEQ_IV_FETCH) @@ -491,6 +492,7 @@ CRT_RPC_DECLARE(crt_iv_fetch, CRT_ISEQ_IV_FETCH, CRT_OSEQ_IV_FETCH) ((d_iov_t) (ivu_sync_type) CRT_VAR) \ /* Bulk handle for iv value */ \ ((crt_bulk_t) (ivu_iv_value_bulk) CRT_VAR) \ + ((d_sg_list_t) (ivu_iv_sgl) CRT_VAR) \ /* Root node for IV UPDATE */ \ ((d_rank_t) (ivu_root_node) CRT_VAR) \ /* Original node that issued crt_iv_update call */ \ @@ -500,7 +502,8 @@ CRT_RPC_DECLARE(crt_iv_fetch, CRT_ISEQ_IV_FETCH, CRT_OSEQ_IV_FETCH) ((uint32_t) (padding) CRT_VAR) #define CRT_OSEQ_IV_UPDATE /* output fields */ \ - ((uint64_t) (rc) CRT_VAR) + ((uint64_t) (rc) CRT_VAR) \ + ((d_sg_list_t) (ivo_iv_sgl) CRT_VAR) CRT_RPC_DECLARE(crt_iv_update, CRT_ISEQ_IV_UPDATE, CRT_OSEQ_IV_UPDATE) @@ -513,8 +516,9 @@ CRT_RPC_DECLARE(crt_iv_update, CRT_ISEQ_IV_UPDATE, CRT_OSEQ_IV_UPDATE) ((d_iov_t) (ivs_key) CRT_VAR) \ /* IOV for sync type */ \ ((d_iov_t) (ivs_sync_type) CRT_VAR) \ + ((d_sg_list_t) (ivs_sync_sgl) CRT_VAR) \ /* IV Class ID */ \ - ((uint32_t) (ivs_class_id) CRT_VAR) + ((uint32_t) (ivs_class_id) CRT_VAR) \ #define CRT_OSEQ_IV_SYNC /* output fields */ \ ((int32_t) (rc) CRT_VAR) diff --git a/src/client/dfs/dfs.c b/src/client/dfs/dfs.c index 5368f605a89..27005c01ade 100644 --- a/src/client/dfs/dfs.c +++ b/src/client/dfs/dfs.c @@ -1026,10 +1026,14 @@ entry_stat(dfs_t *dfs, daos_handle_t th, daos_handle_t oh, const char *name, siz return daos_der2errno(rc); } - rc = daos_obj_query_max_epoch(dir_oh, th, &ep, NULL); - if (rc) { - daos_obj_close(dir_oh, NULL); - return daos_der2errno(rc); + if (dc_supports_epoch_query()) { + rc = daos_obj_query_max_epoch(dir_oh, th, &ep, NULL); + if (rc) { + daos_obj_close(dir_oh, NULL); + return daos_der2errno(rc); + } + } else { + ep = 0; } rc = daos_obj_close(dir_oh, NULL); @@ -4130,10 +4134,14 @@ dfs_lookup_rel_int(dfs_t *dfs, dfs_obj_t *parent, const char *name, int flags, if (stbuf) { daos_epoch_t ep; - rc = daos_obj_query_max_epoch(obj->oh, DAOS_TX_NONE, &ep, NULL); - if (rc) { - daos_obj_close(obj->oh, NULL); - D_GOTO(err_obj, rc = daos_der2errno(rc)); + if (dc_supports_epoch_query()) { + rc = daos_obj_query_max_epoch(obj->oh, DAOS_TX_NONE, &ep, NULL); + if (rc) { + daos_obj_close(obj->oh, NULL); + D_GOTO(err_obj, rc = daos_der2errno(rc)); + } + } else { + ep = 0; } rc = update_stbuf_times(entry, ep, stbuf, NULL); @@ -4658,9 +4666,11 @@ dfs_read_int(dfs_t *dfs, dfs_obj_t *obj, daos_off_t off, dfs_iod_t *iod, if (rc) D_GOTO(err_params, rc); - rc = dc_task_schedule(task, true); - if (rc) - D_GOTO(err_task, rc); + /* + * dc_task_schedule() calls tse_task_complete() even on error (which also calls the + * completion cb that frees params in this case, so we can just ignore the rc here. + */ + dc_task_schedule(task, true); return 0; err_params: diff --git a/src/client/dfuse/dfuse.h b/src/client/dfuse/dfuse.h index f742977763c..ddae9ee0884 100644 --- a/src/client/dfuse/dfuse.h +++ b/src/client/dfuse/dfuse.h @@ -99,9 +99,6 @@ struct dfuse_obj_hdl { ATOMIC uint32_t doh_il_calls; - /** Number of active readdir operations */ - ATOMIC uint32_t doh_readdir_number; - ATOMIC uint64_t doh_write_count; /* Next offset we expect from readdir */ @@ -749,6 +746,9 @@ struct dfuse_inode_entry { struct dfuse_cont *ie_dfs; + /* Lock, used to protect readdir calls */ + pthread_mutex_t ie_lock; + /** Hash table of inodes * All valid inodes are kept in a hash table, using the hash table locking. */ @@ -778,9 +778,6 @@ struct dfuse_inode_entry { /* Readdir handle, if present. May be shared */ struct dfuse_readdir_hdl *ie_rd_hdl; - /** Number of active readdir operations */ - ATOMIC uint32_t ie_readdir_number; - /** file was truncated from 0 to a certain size */ bool ie_truncated; diff --git a/src/client/dfuse/dfuse_core.c b/src/client/dfuse/dfuse_core.c index ab0f6b56a89..662b79e6c21 100644 --- a/src/client/dfuse/dfuse_core.c +++ b/src/client/dfuse/dfuse_core.c @@ -703,7 +703,7 @@ dfuse_cont_open_by_label(struct dfuse_projection_info *fs_handle, struct dfuse_p rc = daos_cont_open(dfp->dfp_poh, label, DAOS_COO_RO, &dfc->dfs_coh, &c_info, NULL); } if (rc == -DER_NONEXIST) { - DFUSE_TRA_INFO(dfc, "daos_cont_open() failed: " DF_RC, DP_RC(rc)); + DFUSE_TRA_INFO(dfc, "daos_cont_open() failed: " DF_RC, DP_RC(rc)); D_GOTO(err_free, rc = daos_der2errno(rc)); } else if (rc != -DER_SUCCESS) { DFUSE_TRA_ERROR(dfc, "daos_cont_open() failed: " DF_RC, DP_RC(rc)); @@ -1111,7 +1111,6 @@ dfuse_open_handle_init(struct dfuse_obj_hdl *oh, struct dfuse_inode_entry *ie) oh->doh_linear_read = true; oh->doh_linear_read_pos = 0; atomic_init(&oh->doh_il_calls, 0); - atomic_init(&oh->doh_readdir_number, 0); atomic_init(&oh->doh_write_count, 0); } @@ -1122,7 +1121,8 @@ dfuse_ie_init(struct dfuse_inode_entry *ie) atomic_init(&ie->ie_open_count, 0); atomic_init(&ie->ie_open_write_count, 0); atomic_init(&ie->ie_il_count, 0); - atomic_init(&ie->ie_readdir_number, 0); + + D_MUTEX_INIT(&ie->ie_lock, NULL); } void @@ -1136,7 +1136,6 @@ dfuse_ie_close(struct dfuse_projection_info *fs_handle, struct dfuse_inode_entry ie->ie_stat.st_ino, ref, ie->ie_name, ie->ie_parent); D_ASSERT(ref == 0); - D_ASSERT(atomic_load_relaxed(&ie->ie_readdir_number) == 0); D_ASSERT(atomic_load_relaxed(&ie->ie_il_count) == 0); D_ASSERT(atomic_load_relaxed(&ie->ie_open_count) == 0); @@ -1156,6 +1155,8 @@ dfuse_ie_close(struct dfuse_projection_info *fs_handle, struct dfuse_inode_entry d_hash_rec_decref(&dfp->dfp_cont_table, &dfc->dfs_entry); } + D_MUTEX_DESTROY(&ie->ie_lock); + D_FREE(ie); } diff --git a/src/client/dfuse/ops/readdir.c b/src/client/dfuse/ops/readdir.c index d6bca1c7c87..f24f62883b9 100644 --- a/src/client/dfuse/ops/readdir.c +++ b/src/client/dfuse/ops/readdir.c @@ -788,11 +788,7 @@ dfuse_cb_readdir(fuse_req_t req, struct dfuse_obj_hdl *oh, size_t size, off_t of char *reply_buff = NULL; int rc = EIO; - D_ASSERTF(atomic_fetch_add_relaxed(&oh->doh_readdir_number, 1) == 0, - "Multiple readdir per handle"); - - D_ASSERTF(atomic_fetch_add_relaxed(&oh->doh_ie->ie_readdir_number, 1) == 0, - "Multiple readdir per inode"); + D_MUTEX_LOCK(&oh->doh_ie->ie_lock); /* Handle the EOD case, the kernel will keep reading until it receives zero replies so * reply early in this case. @@ -820,8 +816,7 @@ dfuse_cb_readdir(fuse_req_t req, struct dfuse_obj_hdl *oh, size_t size, off_t of rc = dfuse_do_readdir(fs_handle, req, oh, reply_buff, &size, offset, plus); out: - atomic_fetch_sub_relaxed(&oh->doh_readdir_number, 1); - atomic_fetch_sub_relaxed(&oh->doh_ie->ie_readdir_number, 1); + D_MUTEX_UNLOCK(&oh->doh_ie->ie_lock); if (rc) DFUSE_REPLY_ERR_RAW(oh, req, rc); diff --git a/src/client/pydaos/__init__.py b/src/client/pydaos/__init__.py index c8e4e1c3359..2f13d83dd8d 100644 --- a/src/client/pydaos/__init__.py +++ b/src/client/pydaos/__init__.py @@ -8,6 +8,7 @@ """ import atexit + from . import pydaos_shim # pylint: disable=relative-beyond-top-level,import-self DAOS_MAGIC = 0x7A8A diff --git a/src/client/pydaos/pydaos_core.py b/src/client/pydaos/pydaos_core.py index ddd845b57d9..401b77a0982 100644 --- a/src/client/pydaos/pydaos_core.py +++ b/src/client/pydaos/pydaos_core.py @@ -11,10 +11,7 @@ import enum # pylint: disable-next=relative-beyond-top-level -from . import pydaos_shim -from . import DAOS_MAGIC -from . import PyDError -from . import DaosClient +from . import DAOS_MAGIC, DaosClient, PyDError, pydaos_shim # Import Object class as an enumeration ObjClassID = enum.Enum( diff --git a/src/client/pydaos/raw/__init__.py b/src/client/pydaos/raw/__init__.py index 5bc0b5872cf..991b711d202 100644 --- a/src/client/pydaos/raw/__init__.py +++ b/src/client/pydaos/raw/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2019-2021 Intel Corporation. +# (C) Copyright 2019-2023 Intel Corporation. # # SPDX-License-Identifier: BSD-2-Clause-Patent # @@ -7,9 +7,10 @@ """ # pylint: disable=wildcard-import -from .conversion import * -from .daos_cref import * -from .daos_api import * +from .conversion import * # noqa: F403 +from .daos_api import * # noqa: F403 +from .daos_cref import * # noqa: F403 + # pylint: enable=wildcard-import -__all__ = ["daos_api", "conversion", "daos_cref"] +__all__ = ["daos_api", "conversion", "daos_cref"] # noqa: F405 diff --git a/src/client/pydaos/raw/daos_api.py b/src/client/pydaos/raw/daos_api.py index 9e4c4f93260..d12d53eb355 100644 --- a/src/client/pydaos/raw/daos_api.py +++ b/src/client/pydaos/raw/daos_api.py @@ -8,19 +8,16 @@ # pylint: disable=consider-using-f-string import ctypes -import threading -import os +import enum import inspect +import os import sys +import threading import time -import enum from .. import pydaos_shim # pylint: disable=relative-beyond-top-level - -from . import daos_cref -from . import conversion from .. import DaosClient - +from . import conversion, daos_cref DaosObjClass = enum.Enum( "DaosObjClass", diff --git a/src/client/setup.py b/src/client/setup.py index 03f8f8ae550..668e7871d47 100644 --- a/src/client/setup.py +++ b/src/client/setup.py @@ -8,9 +8,10 @@ If run from within a compiled DAOS source tree this it will detect the install path automatically, otherwise it'll use the defaults. """ -import os import json -from setuptools import setup, find_packages, Extension +import os + +from setuptools import Extension, find_packages, setup def load_conf(): diff --git a/src/common/dav/out.h b/src/common/dav/out.h index 28bb05b7f73..ebe12044db4 100644 --- a/src/common/dav/out.h +++ b/src/common/dav/out.h @@ -40,8 +40,7 @@ /* produce debug/trace output */ #if defined(DAV_EXTRA_DEBUG) -#define DAV_DBG(fmt, ...) - D_DEBUG(DAV_LOG_FAC, fmt "\n", ## __VA_ARGS__) +#define DAV_DBG(fmt, ...) D_DEBUG(DAV_LOG_FAC, fmt "\n", ##__VA_ARGS__) #else #define DAV_DBG(fmt, ...) SUPPRESS_UNUSED(__VA_ARGS__) #endif diff --git a/src/common/misc.c b/src/common/misc.c index dc910507d98..124b10b21ea 100644 --- a/src/common/misc.c +++ b/src/common/misc.c @@ -235,16 +235,9 @@ daos_sgl_data_len(d_sg_list_t *sgl) daos_size_t daos_sgl_buf_size(d_sg_list_t *sgl) { - daos_size_t size = 0; - int i; - - if (sgl == NULL || sgl->sg_iovs == NULL) + if (sgl == NULL) return 0; - - for (i = 0, size = 0; i < sgl->sg_nr; i++) - size += sgl->sg_iovs[i].iov_buf_len; - - return size; + return d_sgl_buf_size(sgl); } daos_size_t diff --git a/src/container/container_iv.c b/src/container/container_iv.c index cccf8f7d6f2..ff053c14e34 100644 --- a/src/container/container_iv.c +++ b/src/container/container_iv.c @@ -84,10 +84,10 @@ cont_iv_ent_get(struct ds_iv_entry *entry, void **priv) return 0; } -static int +static void cont_iv_ent_put(struct ds_iv_entry *entry, void *priv) { - return 0; + return; } static int @@ -191,8 +191,12 @@ cont_iv_ent_copy(struct ds_iv_entry *entry, struct cont_iv_key *key, cip_acl.dal_ace[src->iv_prop.cip_acl.dal_len]); memcpy(&dst->iv_prop, &src->iv_prop, size); break; + case IV_CONT_AGG_EPOCH_BOUNDRY: + dst->iv_agg_eph.eph = src->iv_agg_eph.eph; + break; default: - D_ERROR("bad iv_class_id %d.\n", entry->iv_class->iv_class_id); + D_ERROR("bad iv_class_id %d: "DF_RC"\n", entry->iv_class->iv_class_id, + DP_RC(-DER_INVAL)); return -DER_INVAL; }; @@ -1071,6 +1075,33 @@ cont_iv_ec_agg_eph_refresh(void *ns, uuid_t cont_uuid, daos_epoch_t eph) IV_CONT_AGG_EPOCH_BOUNDRY); } +int +ds_cont_fetch_ec_agg_boundary(void *ns, uuid_t cont_uuid) +{ + + struct cont_iv_entry iv_entry = { 0 }; + int rc; + + /* Only happens on xstream 0 */ + D_ASSERT(dss_get_module_info()->dmi_xs_id == 0); + uuid_copy(iv_entry.cont_uuid, cont_uuid); + rc = crt_group_rank(NULL, &iv_entry.iv_agg_eph.rank); + if (rc) { + D_ERROR(DF_UUID" crt_group_rank failed "DF_RC"\n", + DP_UUID(cont_uuid), DP_RC(rc)); + return rc; + } + + rc = cont_iv_fetch(ns, IV_CONT_AGG_EPOCH_BOUNDRY, cont_uuid, &iv_entry, + sizeof(struct cont_iv_entry), sizeof(struct cont_iv_entry), + true); + if (rc) + D_ERROR(DF_UUID" cont_iv_fetch failed "DF_RC"\n", + DP_UUID(cont_uuid), DP_RC(rc)); + return rc; + +} + int cont_iv_capability_update(void *ns, uuid_t cont_hdl_uuid, uuid_t cont_uuid, uint64_t flags, uint64_t sec_capas, uint32_t pm_ver) diff --git a/src/container/oid_iv.c b/src/container/oid_iv.c index cf89be761b8..d1041184006 100644 --- a/src/container/oid_iv.c +++ b/src/container/oid_iv.c @@ -210,14 +210,12 @@ oid_iv_ent_get(struct ds_iv_entry *entry, void **_priv) return 0; } -static int +static void oid_iv_ent_put(struct ds_iv_entry *entry, void *priv) { D_ASSERT(priv != NULL); D_DEBUG(DB_TRACE, "%u: ON PUT\n", dss_self_rank()); D_FREE(priv); - - return 0; } static int diff --git a/src/container/srv_container.c b/src/container/srv_container.c index 282761ec2bf..35da023ff24 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -1799,6 +1799,12 @@ cont_agg_eph_leader_ult(void *arg) while (!dss_ult_exiting(svc->cs_ec_leader_ephs_req)) { d_rank_list_t fail_ranks = { 0 }; + if (pool->sp_rebuilding) { + D_DEBUG(DB_MD, DF_UUID "skip during rebuilding.\n", + DP_UUID(pool->sp_uuid)); + goto yield; + } + rc = map_ranks_init(pool->sp_map, PO_COMP_ST_DOWNOUT | PO_COMP_ST_DOWN, &fail_ranks); if (rc) { @@ -1870,6 +1876,8 @@ cont_agg_eph_leader_ult(void *arg) continue; } ec_agg->ea_current_eph = min_eph; + if (pool->sp_rebuilding) + break; } map_ranks_fini(&fail_ranks); diff --git a/src/control/SConscript b/src/control/SConscript index a3fe2c80268..933b090b442 100644 --- a/src/control/SConscript +++ b/src/control/SConscript @@ -1,9 +1,9 @@ """Build DAOS Control Plane""" # pylint: disable=too-many-locals import os -from os.path import join -from os import urandom from binascii import b2a_hex +from os import urandom +from os.path import join def is_firmware_mgmt_build(benv): diff --git a/src/control/cmd/daos_agent/start.go b/src/control/cmd/daos_agent/start.go index 5814c3f5250..791af47ba63 100644 --- a/src/control/cmd/daos_agent/start.go +++ b/src/control/cmd/daos_agent/start.go @@ -48,7 +48,7 @@ type startCmd struct { func (cmd *startCmd) Execute(_ []string) error { if err := common.CheckDupeProcess(); err != nil { - return err + cmd.Notice(err.Error()) } cmd.Infof("Starting %s (pid %d)", versionString(), os.Getpid()) @@ -123,8 +123,7 @@ func (cmd *startCmd) Execute(_ []string) error { drpcSrvStart := time.Now() err = drpcServer.Start(hwlocCtx) if err != nil { - cmd.Errorf("Unable to start socket server on %s: %v", sockPath, err) - return err + return errors.Wrap(err, "unable to start dRPC server") } cmd.Debugf("dRPC socket server started: %s", time.Since(drpcSrvStart)) diff --git a/src/control/drpc/drpc_server.go b/src/control/drpc/drpc_server.go index 6627e4b6f47..35f85ef5758 100644 --- a/src/control/drpc/drpc_server.go +++ b/src/control/drpc/drpc_server.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2018-2022 Intel Corporation. +// (C) Copyright 2018-2023 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -87,28 +87,51 @@ func (d *DomainSocketServer) Listen(ctx context.Context) { // Start sets up the dRPC server socket and kicks off the listener goroutine. func (d *DomainSocketServer) Start(ctx context.Context) error { - // Just in case an old socket file is still lying around - if err := syscall.Unlink(d.sockFile); err != nil && !os.IsNotExist(err) { - return errors.Wrapf(err, "Unable to unlink %s", d.sockFile) + if d == nil { + return errors.New("DomainSocketServer is nil") } addr := &net.UnixAddr{Name: d.sockFile, Net: "unixpacket"} + if err := d.checkExistingSocket(ctx, addr); err != nil { + return err + } + lis, err := net.ListenUnix("unixpacket", addr) if err != nil { - return errors.Wrapf(err, "Unable to listen on unix socket %s", d.sockFile) + return errors.Wrapf(err, "unable to listen on unix socket %s", d.sockFile) } d.listener = lis - // The only writer should be the I/O Engines which should be running as the same user as - // daos_server process. if err := os.Chmod(d.sockFile, d.sockFileMode); err != nil { - return errors.Wrapf(err, "Unable to set permissions on %s", d.sockFile) + return errors.Wrapf(err, "unable to set permissions on %s", d.sockFile) } go d.Listen(ctx) return nil } +func (d *DomainSocketServer) checkExistingSocket(ctx context.Context, addr *net.UnixAddr) error { + conn, err := net.DialUnix("unixpacket", nil, addr) + if err == nil { + _ = conn.Close() + return FaultSocketFileInUse(d.sockFile) + } + + if errors.Is(err, syscall.ENOENT) { + return nil + } + + if errors.Is(err, syscall.ECONNREFUSED) { + // File exists but no one is listening - it's safe to delete. + if err := syscall.Unlink(addr.Name); err != nil && !os.IsNotExist(err) { + return errors.Wrap(err, "unlink old socket file") + } + return nil + } + + return err +} + // RegisterRPCModule takes a Module and associates it with the given // DomainSocketServer so it can be used to process incoming dRPC calls. func (d *DomainSocketServer) RegisterRPCModule(mod Module) { diff --git a/src/control/drpc/drpc_server_test.go b/src/control/drpc/drpc_server_test.go index a63175f95a9..1713a6e8b43 100644 --- a/src/control/drpc/drpc_server_test.go +++ b/src/control/drpc/drpc_server_test.go @@ -17,6 +17,8 @@ import ( "google.golang.org/protobuf/proto" "github.com/daos-stack/daos/src/control/common/test" + "github.com/daos-stack/daos/src/control/fault" + "github.com/daos-stack/daos/src/control/fault/code" "github.com/daos-stack/daos/src/control/logging" ) @@ -144,52 +146,119 @@ func TestNewDomainSocketServer(t *testing.T) { test.AssertEqual(t, dss.sockFile, expectedSock, "wrong sockfile") } -func TestServer_Start_CantUnlinkSocket(t *testing.T) { - log, buf := logging.NewTestLogger(t.Name()) - defer test.ShowBufferOnFailure(t, buf) - - tmpDir, tmpCleanup := test.CreateTestDir(t) - defer tmpCleanup() - - path := filepath.Join(tmpDir, "test.sock") - - // Forbid searching the directory - if err := os.Chmod(tmpDir, 0000); err != nil { - t.Fatalf("Couldn't change permissions on dir: %v", err) +func TestDrpc_DomainSocketServer_Start(t *testing.T) { + sockPath := func(dir string) string { + return filepath.Join(dir, "test.sock") } - defer func() { - _ = os.Chmod(tmpDir, 0700) - }() - - dss, _ := NewDomainSocketServer(log, path, testFileMode) - - err := dss.Start(test.Context(t)) - - test.CmpErr(t, errors.New("unlink"), err) -} -func TestServer_Start_CantListen(t *testing.T) { - log, buf := logging.NewTestLogger(t.Name()) - defer test.ShowBufferOnFailure(t, buf) - - tmpDir, tmpCleanup := test.CreateTestDir(t) - defer tmpCleanup() - - path := filepath.Join(tmpDir, "test.sock") - - // Forbid writing the directory - if err := os.Chmod(tmpDir, 0500); err != nil { - t.Fatalf("Couldn't change permissions on dir: %v", err) + for name, tc := range map[string]struct { + nilServer bool + setup func(t *testing.T, dir string) func() + expErr error + }{ + "nil": { + nilServer: true, + expErr: errors.New("nil"), + }, + "unused existing socket file": { + setup: func(t *testing.T, dir string) func() { + t.Helper() + + f, err := os.Create(sockPath(dir)) + if err != nil { + t.Fatal(err) + } + _ = f.Close() + return func() {} + }, + }, + "can't unlink old socket file": { + setup: func(t *testing.T, dir string) func() { + t.Helper() + + sockFile := sockPath(dir) + f, err := os.Create(sockFile) + if err != nil { + t.Fatal(err) + } + _ = f.Close() + + if err := os.Chmod(dir, 0500); err != nil { + t.Fatalf("Couldn't change permissions on dir: %v", err) + } + return func() { + _ = os.Chmod(dir, 0700) + } + }, + expErr: errors.New("unlink"), + }, + "socket file in use": { + setup: func(t *testing.T, dir string) func() { + t.Helper() + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + other, err := NewDomainSocketServer(log, sockPath(dir), testFileMode) + if err != nil { + t.Fatalf("can't create first server: %s", err.Error()) + } + + err = other.Start(test.Context(t)) + if err != nil { + t.Fatalf("can't start up first server: %s", err.Error()) + } + + // NB: The started server is shut down when the test context is canceled. + return func() {} + }, + expErr: FaultSocketFileInUse(""), + }, + "listen fails": { + setup: func(t *testing.T, dir string) func() { + t.Helper() + + if err := os.Chmod(dir, 0500); err != nil { + t.Fatalf("Couldn't change permissions on dir: %v", err) + } + return func() { + _ = os.Chmod(dir, 0700) + } + }, + expErr: errors.New("listen"), + }, + "success": {}, + } { + t.Run(name, func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + tmpDir, tmpCleanup := test.CreateTestDir(t) + defer tmpCleanup() + + if tc.setup != nil { + teardown := tc.setup(t, tmpDir) + defer teardown() + } + + // Test hack - make sure the right path is included in the fault message for comparison + if fault.IsFaultCode(tc.expErr, code.SocketFileInUse) { + tc.expErr = FaultSocketFileInUse(sockPath(tmpDir)) + } + + var err error + var dss *DomainSocketServer + if !tc.nilServer { + dss, err = NewDomainSocketServer(log, sockPath(tmpDir), testFileMode) + if err != nil { + t.Fatal(err) + } + } + + err = dss.Start(test.Context(t)) + + test.CmpErr(t, tc.expErr, err) + }) } - defer func() { - _ = os.Chmod(tmpDir, 0700) - }() - - dss, _ := NewDomainSocketServer(log, path, testFileMode) - - err := dss.Start(test.Context(t)) - - test.CmpErr(t, errors.New("listen"), err) } func TestServer_RegisterModule(t *testing.T) { diff --git a/src/control/drpc/fault.go b/src/control/drpc/fault.go new file mode 100644 index 00000000000..f41a7882a98 --- /dev/null +++ b/src/control/drpc/fault.go @@ -0,0 +1,27 @@ +// +// (C) Copyright 2023 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package drpc + +import ( + "fmt" + + "github.com/daos-stack/daos/src/control/fault" + "github.com/daos-stack/daos/src/control/fault/code" +) + +// FaultSocketFileInUse indicates that the dRPC socket file was already in use when we tried +// to start the dRPC server. +func FaultSocketFileInUse(path string) *fault.Fault { + return &fault.Fault{ + Domain: "drpc", + Code: code.SocketFileInUse, + Description: fmt.Sprintf("Configured dRPC socket file '%s' is already in use.", path), + Reason: "dRPC socket file already in use", + Resolution: "If another process is using the socket file, configure a different socket directory. " + + "Otherwise, delete the existing socket file and try again.", + } +} diff --git a/src/control/fault/code/codes.go b/src/control/fault/code/codes.go index 89bfb32bed0..4a368aeb5e3 100644 --- a/src/control/fault/code/codes.go +++ b/src/control/fault/code/codes.go @@ -50,6 +50,7 @@ const ( PrivilegedHelperNotPrivileged PrivilegedHelperNotAvailable PrivilegedHelperRequestFailed + SocketFileInUse ) // generic storage fault codes diff --git a/src/control/vendor/github.com/hashicorp/go-msgpack/codec/test.py b/src/control/vendor/github.com/hashicorp/go-msgpack/codec/test.py index 800376f6841..f2c57b06f28 100644 --- a/src/control/vendor/github.com/hashicorp/go-msgpack/codec/test.py +++ b/src/control/vendor/github.com/hashicorp/go-msgpack/codec/test.py @@ -11,7 +11,14 @@ # Ensure all "string" keys are utf strings (else encoded as bytes) -import cbor, msgpack, msgpackrpc, sys, os, threading +import os +import sys +import threading + +import cbor +import msgpack +import msgpackrpc + def get_test_data_list(): # get list with all primitive types, and a combo type diff --git a/src/dtx/dtx_common.c b/src/dtx/dtx_common.c index d2af1b0c06e..5a7f65474dd 100644 --- a/src/dtx/dtx_common.c +++ b/src/dtx/dtx_common.c @@ -1006,6 +1006,7 @@ dtx_renew_epoch(struct dtx_epoch *epoch, struct dtx_handle *dth) { dth->dth_epoch = epoch->oe_value; dth->dth_epoch_bound = dtx_epoch_bound(epoch); + vos_dtx_renew_epoch(dth); } /** @@ -1147,9 +1148,9 @@ dtx_leader_begin(daos_handle_t coh, struct dtx_id *dti, if (rc == 0 && sub_modification_cnt > 0) rc = vos_dtx_attach(dth, false, (flags & DTX_PREPARED) ? true : false); - D_DEBUG(DB_IO, "Start DTX "DF_DTI" sub modification %d, ver %u, leader " + D_DEBUG(DB_IO, "Start DTX "DF_DTI" sub modification %d, ver %u, epoch "DF_X64", leader " DF_UOID", dti_cos_cnt %d, tgt_cnt %d, flags %x: "DF_RC"\n", - DP_DTI(dti), sub_modification_cnt, dth->dth_ver, + DP_DTI(dti), sub_modification_cnt, dth->dth_ver, epoch->oe_value, DP_UOID(*leader_oid), dti_cos_cnt, tgt_cnt, flags, DP_RC(rc)); if (rc != 0) { @@ -1483,9 +1484,9 @@ dtx_begin(daos_handle_t coh, struct dtx_id *dti, rc = vos_dtx_attach(dth, false, false); D_DEBUG(DB_IO, "Start DTX "DF_DTI" sub modification %d, ver %u, " - "dti_cos_cnt %d, flags %x: "DF_RC"\n", + "epoch "DF_X64", dti_cos_cnt %d, flags %x: "DF_RC"\n", DP_DTI(dti), sub_modification_cnt, - dth->dth_ver, dti_cos_cnt, flags, DP_RC(rc)); + dth->dth_ver, epoch->oe_value, dti_cos_cnt, flags, DP_RC(rc)); if (rc != 0) D_FREE(dth); diff --git a/src/engine/server_iv.c b/src/engine/server_iv.c index 29e9218eff4..05542d916b0 100644 --- a/src/engine/server_iv.c +++ b/src/engine/server_iv.c @@ -595,9 +595,13 @@ ivc_on_get(crt_iv_namespace_t ivns, crt_iv_key_t *iv_key, class = entry->iv_class; if (iv_value) { - rc = class->iv_class_ops->ivc_value_alloc(entry, &key, iv_value); - if (rc) - D_GOTO(out, rc); + if (permission & CRT_IV_NO_ALLOC) { + iv_value->sg_nr = 1; + } else { + rc = class->iv_class_ops->ivc_value_alloc(entry, &key, iv_value); + if (rc) + D_GOTO(out, rc); + } } rc = class->iv_class_ops->ivc_ent_get(entry, &entry_priv_val); @@ -626,7 +630,7 @@ ivc_on_get(crt_iv_namespace_t ivns, crt_iv_key_t *iv_key, return rc; } -static int +static void ivc_on_put(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *priv) { struct ds_iv_ns *ns = NULL; @@ -638,7 +642,7 @@ ivc_on_put(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *priv) if (rc != 0) { if (ns != NULL) ds_iv_ns_put(ns); /* balance ivc_on_get */ - return rc; + return; } D_ASSERT(ns != NULL); @@ -652,9 +656,7 @@ ivc_on_put(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *priv) /* Let's deal with iv_value first */ d_sgl_fini(iv_value, true); - rc = entry->iv_class->iv_class_ops->ivc_ent_put(entry, priv_entry->priv); - if (rc) - D_GOTO(put, rc); + entry->iv_class->iv_class_ops->ivc_ent_put(entry, priv_entry->priv); D_FREE(priv_entry); if (--entry->iv_ref > 0) @@ -668,7 +670,7 @@ ivc_on_put(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *priv) ds_iv_ns_put(ns); ds_iv_ns_put(ns); - return rc; + return; } static int diff --git a/src/gurt/misc.c b/src/gurt/misc.c index 003fa68a5ec..4c49b4dd6e6 100644 --- a/src/gurt/misc.c +++ b/src/gurt/misc.c @@ -388,26 +388,27 @@ d_rank_list_alloc(uint32_t size) return rank_list; } -d_rank_list_t * -d_rank_list_realloc(d_rank_list_t *ptr, uint32_t size) +int +d_rank_list_resize(d_rank_list_t *ptr, uint32_t size) { d_rank_t *new_rl_ranks; if (ptr == NULL) - return d_rank_list_alloc(size); + return -DER_INVAL; if (size == 0) { - d_rank_list_free(ptr); - return NULL; + D_FREE(ptr->rl_ranks); + ptr->rl_nr = 0; + return 0; } D_REALLOC_ARRAY(new_rl_ranks, ptr->rl_ranks, ptr->rl_nr, size); if (new_rl_ranks != NULL) { ptr->rl_ranks = new_rl_ranks; ptr->rl_nr = size; } else { - ptr = NULL; + return -DER_NOMEM; } - return ptr; + return 0; } void @@ -430,12 +431,11 @@ d_rank_list_copy(d_rank_list_t *dst, d_rank_list_t *src) } if (dst->rl_nr != src->rl_nr) { - dst = d_rank_list_realloc(dst, src->rl_nr); - if (dst == NULL) { - D_ERROR("d_rank_list_realloc() failed.\n"); - D_GOTO(out, rc = -DER_NOMEM); + rc = d_rank_list_resize(dst, src->rl_nr); + if (rc != 0) { + D_ERROR("d_rank_list_resize() failed.\n"); + D_GOTO(out, rc); } - dst->rl_nr = src->rl_nr; } memcpy(dst->rl_ranks, src->rl_ranks, dst->rl_nr * sizeof(d_rank_t)); @@ -533,10 +533,10 @@ d_rank_list_del(d_rank_list_t *rank_list, d_rank_t rank) D_ASSERT(idx <= new_num); num_bytes = (new_num - idx) * sizeof(d_rank_t); memmove(dest, src, num_bytes); - rank_list = d_rank_list_realloc(rank_list, new_num); - if (rank_list == NULL) { - D_ERROR("d_rank_list_realloc() failed.\n"); - D_GOTO(out, rc = -DER_NOMEM); + rc = d_rank_list_resize(rank_list, new_num); + if (rc != 0) { + D_ERROR("d_rank_list_resize() failed.\n"); + D_GOTO(out, rc); } out: return rc; @@ -545,16 +545,15 @@ d_rank_list_del(d_rank_list_t *rank_list, d_rank_t rank) int d_rank_list_append(d_rank_list_t *rank_list, d_rank_t rank) { - uint32_t old_num = rank_list->rl_nr; - d_rank_list_t *new_rank_list; - int rc = 0; + uint32_t old_num = rank_list->rl_nr; + int rc = 0; - new_rank_list = d_rank_list_realloc(rank_list, old_num + 1); - if (new_rank_list == NULL) { - D_ERROR("d_rank_list_realloc() failed.\n"); - D_GOTO(out, rc = -DER_NOMEM); + rc = d_rank_list_resize(rank_list, old_num + 1); + if (rc != 0) { + D_ERROR("d_rank_list_resize() failed.\n"); + D_GOTO(out, rc); } - new_rank_list->rl_ranks[old_num] = rank; + rank_list->rl_ranks[old_num] = rank; out: return rc; diff --git a/src/include/cart/api.h b/src/include/cart/api.h index 6bf98811e47..402d17f83ed 100644 --- a/src/include/cart/api.h +++ b/src/include/cart/api.h @@ -1632,6 +1632,18 @@ crt_proc_d_rank_list_t(crt_proc_t proc, crt_proc_op_t proc_op, int crt_proc_d_iov_t(crt_proc_t proc, crt_proc_op_t proc_op, d_iov_t *data); +/** + * Generic processing routine. + * + * \param[in,out] proc abstract processor object + * \param[in] proc_op proc operation type + * \param[in,out] data pointer to data + * + * \return DER_SUCCESS on success, negative value if error + */ +int +crt_proc_d_sg_list_t(crt_proc_t proc, crt_proc_op_t proc_op, d_sg_list_t *data); + /** * Create the processor object. * diff --git a/src/include/cart/iv.h b/src/include/cart/iv.h index 1907ae791f7..db5ab93d34e 100644 --- a/src/include/cart/iv.h +++ b/src/include/cart/iv.h @@ -239,6 +239,7 @@ typedef int (*crt_iv_on_hash_cb_t)(crt_iv_namespace_t ivns, typedef enum { CRT_IV_PERM_READ = 0x1, CRT_IV_PERM_WRITE = 0x2, + CRT_IV_NO_ALLOC = 0x4, } crt_iv_perm_t; /** @@ -287,11 +288,8 @@ typedef int (*crt_iv_on_get_cb_t)(crt_iv_namespace_t ivns, * \param[in] iv_value iv_value buffers to return * \param[in] arg private user data * - * \return DER_SUCCESS on success, negative value if error */ -typedef int (*crt_iv_on_put_cb_t)(crt_iv_namespace_t ivns, - d_sg_list_t *iv_value, - void *arg); +typedef void (*crt_iv_on_put_cb_t)(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *arg); /** * Compares two passed iv keys 'key1' and 'key2' and returns either diff --git a/src/include/daos/object.h b/src/include/daos/object.h index 3692c545f4d..3a8d3758858 100644 --- a/src/include/daos/object.h +++ b/src/include/daos/object.h @@ -830,4 +830,12 @@ int dc_obj_enum_unpack(daos_unit_oid_t oid, daos_key_desc_t *kds, int kds_num, d_sg_list_t *sgl, d_iov_t *csum, dc_obj_enum_unpack_cb_t cb, void *cb_arg); + +/** Check if server supports daos_obj_query_max_epoch */ +extern int dc_obj_proto_version; +static inline bool +dc_supports_epoch_query() +{ + return (dc_obj_proto_version >= 9); +} #endif /* __DD_OBJ_H__ */ diff --git a/src/include/daos_srv/container.h b/src/include/daos_srv/container.h index 7f0a5d3390d..3442f4b7de1 100644 --- a/src/include/daos_srv/container.h +++ b/src/include/daos_srv/container.h @@ -256,4 +256,6 @@ void ds_cont_ec_timestamp_update(struct ds_cont_child *cont); typedef int(*cont_rdb_iter_cb_t)(uuid_t pool_uuid, uuid_t cont_uuid, struct rdb_tx *tx, void *arg); int ds_cont_rdb_iterate(struct cont_svc *svc, cont_rdb_iter_cb_t iter_cb, void *cb_arg); int ds_cont_rf_check(uuid_t pool_uuid, uuid_t cont_uuid, struct rdb_tx *tx); + +int ds_cont_fetch_ec_agg_boundary(void *ns, uuid_t cont_uuid); #endif /* ___DAOS_SRV_CONTAINER_H_ */ diff --git a/src/include/daos_srv/dtx_srv.h b/src/include/daos_srv/dtx_srv.h index 864f48b820a..6d0936a5b55 100644 --- a/src/include/daos_srv/dtx_srv.h +++ b/src/include/daos_srv/dtx_srv.h @@ -123,6 +123,7 @@ struct dtx_handle { struct dtx_rsrvd_uint dth_rsrvd_inline; struct dtx_rsrvd_uint *dth_rsrvds; void **dth_deferred; + void *dth_local_stub; /* NVME extents to release */ d_list_t dth_deferred_nvme; /* Committed or comittable DTX list */ diff --git a/src/include/daos_srv/iv.h b/src/include/daos_srv/iv.h index 0895b0c0fa3..75e12481e7c 100644 --- a/src/include/daos_srv/iv.h +++ b/src/include/daos_srv/iv.h @@ -164,9 +164,8 @@ typedef int (*ds_iv_ent_get_t)(struct ds_iv_entry *ent, void **priv); * \param ent [IN] iv class entry to get. * \param priv [IN] private ptr from crt IV callback. * - * \return 0 if succeeds, error code otherwise. */ -typedef int (*ds_iv_ent_put_t)(struct ds_iv_entry *ent, void *priv); +typedef void (*ds_iv_ent_put_t)(struct ds_iv_entry *ent, void *priv); /** * Destroy the data attached to the entry. diff --git a/src/include/daos_srv/vos.h b/src/include/daos_srv/vos.h index 8a749eae1b0..c26cf02f96b 100644 --- a/src/include/daos_srv/vos.h +++ b/src/include/daos_srv/vos.h @@ -852,6 +852,14 @@ vos_update_end(daos_handle_t ioh, uint32_t pm_ver, daos_key_t *dkey, int err, void vos_update_renew_epoch(daos_handle_t ioh, struct dtx_handle *dth); +/** + * Renew the epoch for the DTX entry. + * + * \param dth [IN] Pointer to the DTX handle. + */ +void +vos_dtx_renew_epoch(struct dtx_handle *dth); + /** * Get the recx/epoch list. * diff --git a/src/include/gurt/common.h b/src/include/gurt/common.h index 923be756e02..94b794a6d55 100644 --- a/src/include/gurt/common.h +++ b/src/include/gurt/common.h @@ -406,7 +406,7 @@ void d_rank_list_filter(d_rank_list_t *src_set, d_rank_list_t *dst_set, bool exclude); int d_rank_list_merge(d_rank_list_t *src_set, d_rank_list_t *merge_set); d_rank_list_t *d_rank_list_alloc(uint32_t size); -d_rank_list_t *d_rank_list_realloc(d_rank_list_t *ptr, uint32_t size); +int d_rank_list_resize(d_rank_list_t *ptr, uint32_t size); void d_rank_list_free(d_rank_list_t *rank_list); int d_rank_list_copy(d_rank_list_t *dst, d_rank_list_t *src); void d_rank_list_shuffle(d_rank_list_t *rank_list); @@ -462,6 +462,38 @@ d_sgl_fini(d_sg_list_t *sgl, bool free_iovs) sgl->sg_nr = 0; } +static inline size_t __attribute__((nonnull)) +d_sgl_buf_size(d_sg_list_t *sgl) +{ + size_t size = 0; + uint32_t i; + + if (sgl->sg_iovs == NULL) + return 0; + + for (i = 0, size = 0; i < sgl->sg_nr; i++) + size += sgl->sg_iovs[i].iov_buf_len; + + return size; +} + +static inline void +d_sgl_buf_copy(d_sg_list_t *dst_sgl, d_sg_list_t *src_sgl) +{ + uint32_t i; + + D_ASSERT(dst_sgl->sg_nr >= src_sgl->sg_nr); + for (i = 0; i < src_sgl->sg_nr; i++) { + D_ASSERT(dst_sgl->sg_iovs[i].iov_buf_len >= + src_sgl->sg_iovs[i].iov_buf_len); + + memcpy(dst_sgl->sg_iovs[i].iov_buf, src_sgl->sg_iovs[i].iov_buf, + src_sgl->sg_iovs[i].iov_buf_len); + dst_sgl->sg_iovs[i].iov_len = src_sgl->sg_iovs[i].iov_len; + dst_sgl->sg_iovs[i].iov_buf_len = src_sgl->sg_iovs[i].iov_buf_len; + } +} + void d_getenv_bool(const char *env, bool *bool_val); void d_getenv_char(const char *env, char *char_val); void d_getenv_int(const char *env, unsigned int *int_val); diff --git a/src/object/obj_rpc.c b/src/object/obj_rpc.c index 0137bc48dbc..bfff5544b40 100644 --- a/src/object/obj_rpc.c +++ b/src/object/obj_rpc.c @@ -491,52 +491,6 @@ crt_proc_struct_obj_iod_array(crt_proc_t proc, crt_proc_op_t proc_op, return rc; } -static int -crt_proc_d_sg_list_t(crt_proc_t proc, crt_proc_op_t proc_op, d_sg_list_t *p) -{ - int i; - int rc; - - if (FREEING(proc_op)) { - /* NB: don't need free in crt_proc_d_iov_t() */ - D_FREE(p->sg_iovs); - return 0; - } - - rc = crt_proc_uint32_t(proc, proc_op, &p->sg_nr); - if (unlikely(rc)) - return rc; - - rc = crt_proc_uint32_t(proc, proc_op, &p->sg_nr_out); - if (unlikely(rc)) - return rc; - - if (p->sg_nr == 0) - return 0; - - switch (proc_op) { - case CRT_PROC_DECODE: - D_ALLOC_ARRAY(p->sg_iovs, p->sg_nr); - if (p->sg_iovs == NULL) - return -DER_NOMEM; - /* fall through to fill sg_iovs */ - case CRT_PROC_ENCODE: - for (i = 0; i < p->sg_nr; i++) { - rc = crt_proc_d_iov_t(proc, proc_op, &p->sg_iovs[i]); - if (unlikely(rc)) { - if (DECODING(proc_op)) - D_FREE(p->sg_iovs); - return rc; - } - } - break; - default: - return -DER_INVAL; - } - - return rc; -} - static int crt_proc_struct_daos_shard_tgt(crt_proc_t proc, crt_proc_op_t proc_op, struct daos_shard_tgt *p) diff --git a/src/object/srv_obj.c b/src/object/srv_obj.c index 5b35647dc54..334dae8b4da 100644 --- a/src/object/srv_obj.c +++ b/src/object/srv_obj.c @@ -1785,7 +1785,7 @@ obj_local_rw_internal(crt_rpc_t *rpc, struct obj_io_context *ioc, daos_iod_t *io vos_update_renew_epoch(ioh, dth); D_DEBUG(DB_IO, - "update rpc %p renew epoch "DF_U64" => "DF_U64" for "DF_DTI"\n", + "update rpc %p renew epoch "DF_X64" => "DF_X64" for "DF_DTI"\n", rpc, orw->orw_epoch, dth->dth_epoch, DP_DTI(&orw->orw_dti)); orw->orw_epoch = dth->dth_epoch; @@ -3483,8 +3483,9 @@ static int obj_local_punch(struct obj_punch_in *opi, crt_opcode_t opc, struct obj_io_context *ioc, struct dtx_handle *dth) { - struct ds_cont_child *cont = ioc->ioc_coc; - int rc = 0; + struct ds_cont_child *cont = ioc->ioc_coc; + uint64_t sched_seq; + int rc = 0; if (daos_is_zero_dti(&opi->opi_dti)) { D_DEBUG(DB_TRACE, "disable dtx\n"); @@ -3496,6 +3497,8 @@ obj_local_punch(struct obj_punch_in *opi, crt_opcode_t opc, if (rc != 0) goto out; + sched_seq = sched_cur_seq(); + switch (opc) { case DAOS_OBJ_RPC_PUNCH: case DAOS_OBJ_RPC_TGT_PUNCH: @@ -3528,8 +3531,60 @@ obj_local_punch(struct obj_punch_in *opi, crt_opcode_t opc, if (dth != NULL && obj_dtx_need_refresh(dth, rc)) { rc = dtx_refresh(dth, ioc->ioc_coc); - if (rc == -DER_AGAIN) + if (rc != -DER_AGAIN) + goto out; + + if (unlikely(sched_cur_seq() == sched_seq)) goto again; + + /* + * There is CPU yield after DTX start, and the resent RPC may be handled + * during that. Let's check resent again before further process. + */ + + if (dth->dth_need_validation) { + daos_epoch_t epoch = 0; + int rc1; + + rc1 = dtx_handle_resend(ioc->ioc_vos_coh, &opi->opi_dti, &epoch, NULL); + switch (rc1) { + case 0: + opi->opi_epoch = epoch; + /* Fall through */ + case -DER_ALREADY: + rc = -DER_ALREADY; + break; + case -DER_NONEXIST: + case -DER_EP_OLD: + break; + default: + rc = rc1; + break; + } + } + + /* + * For solo punch, it will be handled via one-phase transaction. If there is CPU + * yield after its epoch generated, we will renew the epoch, then we can use the + * epoch to sort related solo DTXs based on their epochs. + */ + if (rc == -DER_AGAIN && dth->dth_solo) { + struct dtx_epoch epoch; + + epoch.oe_value = d_hlc_get(); + epoch.oe_first = epoch.oe_value; + epoch.oe_flags = orf_to_dtx_epoch_flags(opi->opi_flags); + + dtx_renew_epoch(&epoch, dth); + + D_DEBUG(DB_IO, + "punch rpc %u renew epoch "DF_X64" => "DF_X64" for "DF_DTI"\n", + opc, opi->opi_epoch, dth->dth_epoch, DP_DTI(&opi->opi_dti)); + + opi->opi_epoch = dth->dth_epoch; + } + + goto again; } out: @@ -4463,10 +4518,11 @@ ds_cpd_handle_one(crt_rpc_t *rpc, struct daos_cpd_sub_head *dcsh, dcsh->dcsh_epoch.oe_value = d_hlc_get(); dtx_renew_epoch(&dcsh->dcsh_epoch, dth); - vos_update_renew_epoch(iohs[0], dth); + if (daos_handle_is_valid(iohs[0])) + vos_update_renew_epoch(iohs[0], dth); D_DEBUG(DB_IO, - "CPD rpc %p renew epoch "DF_U64" => "DF_U64" for "DF_DTI"\n", + "CPD rpc %p renew epoch "DF_X64" => "DF_X64" for "DF_DTI"\n", rpc, epoch, dcsh->dcsh_epoch.oe_value, DP_DTI(&dcsh->dcsh_xid)); } } diff --git a/src/object/srv_obj_migrate.c b/src/object/srv_obj_migrate.c index 5ff44520bc1..4d22c1bb228 100644 --- a/src/object/srv_obj_migrate.c +++ b/src/object/srv_obj_migrate.c @@ -928,7 +928,8 @@ migrate_update_parity(struct migrate_one *mrone, daos_epoch_t parity_eph, static int __migrate_fetch_update_parity(struct migrate_one *mrone, daos_handle_t oh, - daos_iod_t *iods, daos_epoch_t **ephs, uint32_t iods_num, + daos_iod_t *iods, daos_epoch_t fetch_eph, + daos_epoch_t **ephs, uint32_t iods_num, struct ds_cont_child *ds_cont, bool encode) { d_sg_list_t sgls[OBJ_ENUM_UNPACK_MAX_IODS]; @@ -959,7 +960,7 @@ __migrate_fetch_update_parity(struct migrate_one *mrone, daos_handle_t oh, D_DEBUG(DB_REBUILD, DF_UOID" mrone %p dkey "DF_KEY" nr %d eph "DF_U64"\n", DP_UOID(mrone->mo_oid), mrone, DP_KEY(&mrone->mo_dkey), iods_num, mrone->mo_epoch); - rc = mrone_obj_fetch(mrone, oh, sgls, iods, iods_num, mrone->mo_epoch, DIOF_FOR_MIGRATION, + rc = mrone_obj_fetch(mrone, oh, sgls, iods, iods_num, fetch_eph, DIOF_FOR_MIGRATION, NULL); if (rc) { D_ERROR("migrate dkey "DF_KEY" failed: "DF_RC"\n", @@ -1022,22 +1023,54 @@ static int migrate_fetch_update_parity(struct migrate_one *mrone, daos_handle_t oh, struct ds_cont_child *ds_cont) { + int i; + int j; int rc = 0; /* If it is parity recxs from another replica, then let's encode it anyway */ - if (mrone->mo_iods_num_from_parity > 0) { - rc = __migrate_fetch_update_parity(mrone, oh, mrone->mo_iods_from_parity, - mrone->mo_iods_update_ephs_from_parity, - mrone->mo_iods_num_from_parity, ds_cont, - true); + for (i = 0; i < mrone->mo_iods_num_from_parity; i++) { + for (j = 0; j < mrone->mo_iods_from_parity[i].iod_nr; j++) { + daos_iod_t iod = mrone->mo_iods_from_parity[i]; + daos_epoch_t fetch_eph; + daos_epoch_t update_eph; + daos_epoch_t *update_eph_p; - if (rc) - return rc; + iod.iod_nr = 1; + iod.iod_recxs = &mrone->mo_iods_from_parity[i].iod_recxs[j]; + /* If the epoch is higher than EC aggregate boundary, then + * it should use stable epoch to fetch the data, since + * the data could be aggregated independently on parity + * and data shard, so using stable epoch could make sure + * the consistency view during rebuild. And also EC aggregation + * should already aggregate the parity, so there should not + * be any partial update on the parity as well. + * + * Otherwise there might be partial update on this rebuilding + * shard, so let's use the epoch from the parity shard to fetch + * the data here, which will make sure partial update will not + * be fetched here. And also EC aggregation is being disabled + * at the moment, so there should not be any vos aggregation + * impact this process as well. + */ + if (ds_cont->sc_ec_agg_eph_boundary > + mrone->mo_iods_update_ephs_from_parity[i][j]) + fetch_eph = mrone->mo_epoch; + else + fetch_eph = mrone->mo_iods_update_ephs_from_parity[i][j]; + + update_eph = mrone->mo_iods_update_ephs_from_parity[i][j]; + update_eph_p = &update_eph; + rc = __migrate_fetch_update_parity(mrone, oh, &iod, fetch_eph, &update_eph_p, + mrone->mo_iods_num_from_parity, ds_cont, + true); + if (rc) + return rc; + } } /* Otherwise, keep it as replicate recx */ if (mrone->mo_iod_num > 0) { - rc = __migrate_fetch_update_parity(mrone, oh, mrone->mo_iods, + rc = __migrate_fetch_update_parity(mrone, oh, mrone->mo_iods, mrone->mo_epoch, mrone->mo_iods_update_ephs, mrone->mo_iod_num, ds_cont, false); } @@ -1350,21 +1383,39 @@ migrate_fetch_update_bulk(struct migrate_one *mrone, daos_handle_t oh, /* For EC object, if the migration include both extent from parity rebuild * and extent from replicate rebuild, let rebuild the extent with parity first, * then extent from replication. - * - * Since the parity shard epoch should be higher or equal to the data shard epoch, - * so let's use the minimum epochs of all parity shards as the update epoch of - * this data shard. */ - for (i = 0; i < mrone->mo_iods_num_from_parity; i++) { for (j = 0; j < mrone->mo_iods_from_parity[i].iod_nr; j++) { daos_iod_t iod = mrone->mo_iods_from_parity[i]; + daos_epoch_t fetch_eph; + daos_epoch_t update_eph; iod.iod_nr = 1; iod.iod_recxs = &mrone->mo_iods_from_parity[i].iod_recxs[j]; - rc = __migrate_fetch_update_bulk(mrone, oh, &iod, 1, - mrone->mo_iods_update_ephs_from_parity[i][j], - mrone->mo_iods_update_ephs_from_parity[i][j], + + /* If the epoch is higher than EC aggregate boundary, then + * it should use stable epoch to fetch the data, since + * the data could be aggregated independently on parity + * and data shard, so using stable epoch could make sure + * the consistency view during rebuild. And also EC aggregation + * should already aggregate the parity, so there should not + * be any partial update on the parity as well. + * + * Otherwise there might be partial update on this rebuilding + * shard, so let's use the epoch from the parity shard to fetch + * the data here, which will make sure partial update will not + * be fetched here. And also EC aggregation is being disabled + * at the moment, so there should not be any vos aggregation + * impact this process as well. + */ + if (ds_cont->sc_ec_agg_eph_boundary > + mrone->mo_iods_update_ephs_from_parity[i][j]) + fetch_eph = mrone->mo_epoch; + else + fetch_eph = mrone->mo_iods_update_ephs_from_parity[i][j]; + + update_eph = mrone->mo_iods_update_ephs_from_parity[i][j]; + rc = __migrate_fetch_update_bulk(mrone, oh, &iod, 1, fetch_eph, update_eph, DIOF_EC_RECOV_FROM_PARITY, ds_cont); if (rc != 0) D_GOTO(out, rc); @@ -3165,6 +3216,16 @@ migrate_cont_iter_cb(daos_handle_t ih, d_iov_t *key_iov, D_GOTO(out_put, rc); } + rc = ds_cont_fetch_ec_agg_boundary(dp->sp_iv_ns, cont_uuid); + if (rc) { + /* Sometime it may too early to fetch the EC boundary, + * since EC boundary does not start yet, which is forbidden + * during rebuild anyway, so let's continue. + */ + D_DEBUG(DB_REBUILD, DF_UUID" fetch agg_boundary failed: "DF_RC"\n", + DP_UUID(cont_uuid), DP_RC(rc)); + } + arg.yield_freq = DEFAULT_YIELD_FREQ; arg.cont_root = root; arg.snaps = snapshots; diff --git a/src/pool/srv_iv.c b/src/pool/srv_iv.c index 53908a490fc..3eaa05ce48e 100644 --- a/src/pool/srv_iv.c +++ b/src/pool/srv_iv.c @@ -613,10 +613,10 @@ pool_iv_ent_get(struct ds_iv_entry *entry, void **priv) return 0; } -static int +static void pool_iv_ent_put(struct ds_iv_entry *entry, void *priv) { - return 0; + return; } static int diff --git a/src/rdb/raft b/src/rdb/raft index c354cd7fb1e..efa15f46360 160000 --- a/src/rdb/raft +++ b/src/rdb/raft @@ -1 +1 @@ -Subproject commit c354cd7fb1e1dbf3fbc9b3d24a6b05d0c5c9d5af +Subproject commit efa15f46360078ff427562c53d23ed6f0e4a06ac diff --git a/src/rdb/raft_tests/raft_tests.py b/src/rdb/raft_tests/raft_tests.py index f7ca5acd4be..0e701f4fd7c 100755 --- a/src/rdb/raft_tests/raft_tests.py +++ b/src/rdb/raft_tests/raft_tests.py @@ -8,10 +8,10 @@ Makefile. Check the output for the number of "not ok" occurrences and return this number as the return code. ''' +import json +import os import subprocess # nosec import sys -import os -import json TEST_NOT_RUN = -1 DIR = os.path.join(os.path.dirname(os.path.relpath(os.path.dirname(__file__))), 'raft') diff --git a/src/rdb/rdb_raft.c b/src/rdb/rdb_raft.c index cfc292c0c6a..01af06519b9 100644 --- a/src/rdb/rdb_raft.c +++ b/src/rdb/rdb_raft.c @@ -1093,18 +1093,10 @@ rdb_raft_update_node(struct rdb *db, uint64_t index, raft_entry_t *entry) goto out_replicas; } - if (entry->type == RAFT_LOGTYPE_ADD_NODE) { + if (entry->type == RAFT_LOGTYPE_ADD_NODE) rc = d_rank_list_append(replicas, rank); - } else if (entry->type == RAFT_LOGTYPE_REMOVE_NODE) { - /* never expect 1->0 in practice, right? But protect against double-free. */ - bool replicas_freed = (replicas->rl_nr == 1); - + else if (entry->type == RAFT_LOGTYPE_REMOVE_NODE) rc = d_rank_list_del(replicas, rank); - if (replicas_freed) { - D_ASSERT(rc == -DER_NOMEM); - replicas = NULL; - } - } if (rc != 0) goto out_replicas; diff --git a/src/rebuild/rebuild_iv.c b/src/rebuild/rebuild_iv.c index 15a86ceb5fc..da306be72a6 100644 --- a/src/rebuild/rebuild_iv.c +++ b/src/rebuild/rebuild_iv.c @@ -66,10 +66,10 @@ rebuild_iv_ent_get(struct ds_iv_entry *entry, void **priv) return 0; } -static int +static void rebuild_iv_ent_put(struct ds_iv_entry *entry, void *priv) { - return 0; + return; } static int diff --git a/src/rebuild/srv.c b/src/rebuild/srv.c index fca94bf4105..1cb174e8d18 100644 --- a/src/rebuild/srv.c +++ b/src/rebuild/srv.c @@ -1093,7 +1093,7 @@ rebuild_debug_print_queue() * This only accumulates the targets in a single task, so it doesn't * need to be very big. 200 bytes is enough for ~30 5-digit target ids */ - char tgts_buf[200]; + char tgts_buf[200] = { 0 }; int i; /* Position in stack buffer where str data should be written next */ size_t tgts_pos; @@ -1121,7 +1121,7 @@ rebuild_debug_print_queue() } D_DEBUG(DB_REBUILD, DF_UUID" op=%s ver=%u tgts=%s\n", DP_UUID(task->dst_pool_uuid), RB_OP_STR(task->dst_rebuild_op), - task->dst_map_ver, tgts_buf); + task->dst_map_ver, task->dst_tgts.pti_number > 0 ? tgts_buf : "None"); } } diff --git a/src/tests/ftest/__init__.py b/src/tests/ftest/__init__.py index bbb94ab2c77..b9ad76b7b8f 100644 --- a/src/tests/ftest/__init__.py +++ b/src/tests/ftest/__init__.py @@ -1,3 +1,4 @@ """ftest __init__.py.""" import pkg_resources + pkg_resources.declare_namespace(__name__) diff --git a/src/tests/ftest/aggregation/basic.py b/src/tests/ftest/aggregation/basic.py index 3ead6abfbbf..61b389faaad 100644 --- a/src/tests/ftest/aggregation/basic.py +++ b/src/tests/ftest/aggregation/basic.py @@ -5,6 +5,7 @@ """ import time + from ior_test_base import IorTestBase diff --git a/src/tests/ftest/aggregation/dfuse_space_check.py b/src/tests/ftest/aggregation/dfuse_space_check.py index 824c42af002..f95025328f3 100644 --- a/src/tests/ftest/aggregation/dfuse_space_check.py +++ b/src/tests/ftest/aggregation/dfuse_space_check.py @@ -4,8 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import os +import time + from ior_test_base import IorTestBase diff --git a/src/tests/ftest/aggregation/io_small.py b/src/tests/ftest/aggregation/io_small.py index ebb13d98352..6dd84cf3dbc 100644 --- a/src/tests/ftest/aggregation/io_small.py +++ b/src/tests/ftest/aggregation/io_small.py @@ -5,8 +5,9 @@ """ import time -from ior_test_base import IorTestBase + from general_utils import human_to_bytes +from ior_test_base import IorTestBase class DaosAggregationIOSmall(IorTestBase): diff --git a/src/tests/ftest/aggregation/multiple_pool_cont.py b/src/tests/ftest/aggregation/multiple_pool_cont.py index e6b6b2f7968..41f1a19329c 100644 --- a/src/tests/ftest/aggregation/multiple_pool_cont.py +++ b/src/tests/ftest/aggregation/multiple_pool_cont.py @@ -5,9 +5,9 @@ """ import time -from ior_utils import run_ior -from ior_test_base import IorTestBase from exception_utils import CommandFailure +from ior_test_base import IorTestBase +from ior_utils import run_ior from job_manager_utils import get_job_manager diff --git a/src/tests/ftest/aggregation/punching.py b/src/tests/ftest/aggregation/punching.py index c347f0ee284..b95484a778b 100644 --- a/src/tests/ftest/aggregation/punching.py +++ b/src/tests/ftest/aggregation/punching.py @@ -5,6 +5,7 @@ """ import time + from mdtest_test_base import MdtestBase diff --git a/src/tests/ftest/aggregation/shutdown_restart.py b/src/tests/ftest/aggregation/shutdown_restart.py index b421b0efab1..bea3c2e9eae 100644 --- a/src/tests/ftest/aggregation/shutdown_restart.py +++ b/src/tests/ftest/aggregation/shutdown_restart.py @@ -5,8 +5,8 @@ """ import time -from ior_test_base import IorTestBase from dmg_utils import check_system_query_status +from ior_test_base import IorTestBase class IoAggregation(IorTestBase): diff --git a/src/tests/ftest/aggregation/throttling.py b/src/tests/ftest/aggregation/throttling.py index e343df82912..f40de713773 100644 --- a/src/tests/ftest/aggregation/throttling.py +++ b/src/tests/ftest/aggregation/throttling.py @@ -5,6 +5,7 @@ """ import time + from ior_test_base import IorTestBase from ior_utils import IorCommand, IorMetrics diff --git a/src/tests/ftest/avocado_tests.py b/src/tests/ftest/avocado_tests.py index 0a2c12a8859..42092e3588c 100644 --- a/src/tests/ftest/avocado_tests.py +++ b/src/tests/ftest/avocado_tests.py @@ -5,6 +5,7 @@ """ import time + from apricot import Test diff --git a/src/tests/ftest/cart/iv/iv_one_node.py b/src/tests/ftest/cart/iv/iv_one_node.py index e566d5d36ff..42b2b8380de 100644 --- a/src/tests/ftest/cart/iv/iv_one_node.py +++ b/src/tests/ftest/cart/iv/iv_one_node.py @@ -3,14 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import time -import tempfile +import codecs import json import os +import shlex import struct -import codecs import subprocess # nosec -import shlex +import tempfile +import time import traceback from cart_utils import CartTest diff --git a/src/tests/ftest/cart/iv/iv_two_node.py b/src/tests/ftest/cart/iv/iv_two_node.py index 4c49e9305a9..0ca61fe10b1 100644 --- a/src/tests/ftest/cart/iv/iv_two_node.py +++ b/src/tests/ftest/cart/iv/iv_two_node.py @@ -3,14 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import time -import tempfile +import codecs import json import os +import shlex import struct -import codecs import subprocess # nosec -import shlex +import tempfile +import time from cart_utils import CartTest diff --git a/src/tests/ftest/cart/iv_server.c b/src/tests/ftest/cart/iv_server.c index 2e6488bf7aa..43927040a9a 100644 --- a/src/tests/ftest/cart/iv_server.c +++ b/src/tests/ftest/cart/iv_server.c @@ -623,7 +623,7 @@ iv_on_get(crt_iv_namespace_t ivns, crt_iv_key_t *iv_key, return 0; } -static int +static void iv_on_put(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *user_priv) { DBG_ENTRY(); @@ -635,8 +635,6 @@ iv_on_put(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *user_priv) dump_all_keys("ON_PUTVALUE"); DBG_EXIT(); - - return 0; } static void diff --git a/src/tests/ftest/cart/util/cart_logparse.py b/src/tests/ftest/cart/util/cart_logparse.py index 63cb9f368f6..cfe74ce8b4e 100644 --- a/src/tests/ftest/cart/util/cart_logparse.py +++ b/src/tests/ftest/cart/util/cart_logparse.py @@ -10,10 +10,10 @@ This provides a way of querying CaRT logfiles for processing. """ -from collections import OrderedDict import bz2 import os import re +from collections import OrderedDict class InvalidPid(Exception): diff --git a/src/tests/ftest/cart/util/cart_logtest.py b/src/tests/ftest/cart/util/cart_logtest.py index 09a3a6fc9c6..a76b30653b8 100755 --- a/src/tests/ftest/cart/util/cart_logtest.py +++ b/src/tests/ftest/cart/util/cart_logtest.py @@ -6,13 +6,14 @@ """This provides consistency checking for CaRT log files.""" +import argparse import re import sys import time -import argparse -from collections import OrderedDict, Counter +from collections import Counter, OrderedDict import cart_logparse + HAVE_TABULATE = True try: import tabulate diff --git a/src/tests/ftest/cart/util/cart_logusage.py b/src/tests/ftest/cart/util/cart_logusage.py new file mode 100644 index 00000000000..358718141de --- /dev/null +++ b/src/tests/ftest/cart/util/cart_logusage.py @@ -0,0 +1,199 @@ +# (C) Copyright 2023 Intel Corporation +# +# SPDX-License-Identifier: BSD-2-Clause-Patent + +"""Mark code usage from daos logs + +Track what lines of code has caused logging and use this to generate xml 'coverage' data which +can be rendered in Jenkins and used to identify areas which are not exercised. + +Registered as a callback for all log tracing but saves results across the entire run. +""" + +import json +import os + + +class CodeLoc(): + """Logging data for single code location""" + + def __init__(self, line=None): + self.lineno = 0 + self.count = 0 + self.fault_injected = False + self.no_fault = False + self.allocation = False + if line: + self.lineno = line.lineno + self.allocation = (line.is_calloc() or line.is_realloc()) + self.add(line) + + def add(self, line): + """Record an extra logging instance for this location""" + self.count += 1 + if line.is_fi_site(): + self.fault_injected = True + else: + self.no_fault = True + + def is_fi_location(self): + """Return true if a possible fault injection site""" + return (self.allocation or self.fault_injected) + + def counts(self): + """Return a tuple of (count, possible-count)""" + if not self.is_fi_location(): + return (1, 1) + count = 0 + if self.no_fault: + count += 1 + if self.fault_injected: + count += 1 + return (count, 2) + + def xml_str(self): + """Return a xml string for this line""" + if not self.is_fi_location(): + return f' \n' + + condc = 'condition-coverage="50% (1/2)"' + + taken = '0%' + not_taken = '0%' + if self.no_fault: + taken = '100%' + if self.fault_injected: + not_taken = '100%' + if self.no_fault: + condc = 'condition-coverage="100% (2/2)"' + + return f""" + + + + + +""" + + +class UsageTracer(): + """Save what lines are executed""" + + def __init__(self): + self._files = {} + + def report(self): + """Report per log""" + return + + def load(self, fname): + """Load intermediate data from file""" + with open(fname, 'r') as fd: + idata = json.load(fd) + + data = {} + for (key, value) in idata.items(): + data[key] = {} + # Iterate over files. + for (key2, value2) in value.items(): + data[key][key2] = {} + # Iterate over line numbers. + for (key3, value3) in value2.items(): + new_obj = CodeLoc() + new_obj.lineno = key3 + new_obj.count = value3[0] + new_obj.allocation = value3[1] + new_obj.fault_injected = value3[2] + new_obj.no_fault = value3[3] + new_obj.lineno = key3 + data[key][key2][key3] = new_obj + self._files = data + + def save(self, fname): + """Save intermediate data to file""" + data = {} + # Iterate over directories. + for (key, value) in self._files.items(): + data[key] = {} + # Iterate over files. + for (key2, value2) in value.items(): + data[key][key2] = {} + # Iterate over line numbers. + for (key3, value3) in value2.items(): + data[key][key2][key3] = [value3.count, value3.allocation, + value3.fault_injected, value3.no_fault] + + with open(fname, 'w') as fd: + json.dump(data, fd) + + def report_all(self, fname): + """Report on everything""" + if not self._files: + return + with open(fname, 'w') as fd: + self._save(fd) + + def _save(self, fd): + + fd.write("\n") + fd.write('\n") + fd.write('\n') + fd.write(""" +. + +\n""") + + for (dname, bname) in self._files.items(): + # Patch up two areas of logging in the go code that mis-report source code names. + if not dname.startswith('src'): + if dname == '': + dname = 'src/control/cmd/daos' + else: + parts = dname.split('/') + while parts[0] != 'src': + parts.pop(0) + dname = '/'.join(parts) + + fd.write(f'\n') + fd.write('\n') + + for data in bname: + taken = 0 + possible = 0 + xml = '' + for loc in bname[data].values(): + (ttt, ptt) = loc.counts() + taken += ttt + possible += ptt + xml += loc.xml_str() + rate = taken / possible + fd.write( + f' \n') + fd.write(' \n') + fd.write(' \n') + fd.write(xml) + fd.write(' \n') + fd.write(' \n') + fd.write('\n') + fd.write('\n') + fd.write('\n') + + def add_line(self, line): + """Register a line""" + try: + fname = line.filename + except AttributeError: + return + + dname = os.path.dirname(fname) + bname = os.path.basename(fname) + if dname not in self._files: + self._files[dname] = {} + if bname not in self._files[dname]: + self._files[dname][bname] = {} + lineno = line.lineno + if lineno in self._files[dname][bname]: + self._files[dname][bname][lineno].add(line) + else: + self._files[dname][bname][lineno] = CodeLoc(line=line) diff --git a/src/tests/ftest/cart/util/cart_utils.py b/src/tests/ftest/cart/util/cart_utils.py index cb6712fb55a..5bed4b31617 100644 --- a/src/tests/ftest/cart/util/cart_utils.py +++ b/src/tests/ftest/cart/util/cart_utils.py @@ -3,23 +3,22 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import time +import glob +import logging import os +import re import shlex import subprocess # nosec -import logging -import re -import glob - -from ClusterShell.NodeSet import NodeSet +import time import cart_logparse import cart_logtest from apricot import TestWithoutServers -from run_utils import stop_processes +from ClusterShell.NodeSet import NodeSet from host_utils import get_local_host -from write_host_file import write_host_file from job_manager_utils import Orterun +from run_utils import stop_processes +from write_host_file import write_host_file class CartTest(TestWithoutServers): diff --git a/src/tests/ftest/checksum/csum_basic.py b/src/tests/ftest/checksum/csum_basic.py index bbb396b9f3b..7b4cee25bcd 100644 --- a/src/tests/ftest/checksum/csum_basic.py +++ b/src/tests/ftest/checksum/csum_basic.py @@ -5,10 +5,10 @@ """ import ctypes -from pydaos.raw import (DaosContainer, IORequest, - DaosObj) + from apricot import TestWithServers from general_utils import create_string_buffer +from pydaos.raw import DaosContainer, DaosObj, IORequest class CsumContainerValidation(TestWithServers): diff --git a/src/tests/ftest/checksum/csum_error_logging.py b/src/tests/ftest/checksum/csum_error_logging.py index d3de374da56..9569f2e31ea 100644 --- a/src/tests/ftest/checksum/csum_error_logging.py +++ b/src/tests/ftest/checksum/csum_error_logging.py @@ -5,7 +5,6 @@ """ from avocado import fail_on - from daos_core_base import DaosCoreBase from dmg_utils import get_dmg_smd_info from exception_utils import CommandFailure diff --git a/src/tests/ftest/config_file_gen.py b/src/tests/ftest/config_file_gen.py index c0031d0e4bd..58f97f3b902 100755 --- a/src/tests/ftest/config_file_gen.py +++ b/src/tests/ftest/config_file_gen.py @@ -10,16 +10,13 @@ import logging import sys - from argparse import ArgumentParser, RawDescriptionHelpFormatter + +from util.agent_utils_params import DaosAgentTransportCredentials, DaosAgentYamlParameters from util.command_utils_base import CommonConfig +from util.dmg_utils_params import DmgTransportCredentials, DmgYamlParameters from util.exception_utils import CommandFailure -from util.agent_utils_params import \ - DaosAgentYamlParameters, DaosAgentTransportCredentials -from util.server_utils_params import \ - DaosServerYamlParameters, DaosServerTransportCredentials -from util.dmg_utils_params import \ - DmgYamlParameters, DmgTransportCredentials +from util.server_utils_params import DaosServerTransportCredentials, DaosServerYamlParameters def generate_agent_config(args): @@ -53,7 +50,7 @@ def generate_server_config(args): config = DaosServerYamlParameters(args.server_file, common_cfg) config.engine_params[0].storage.storage_tiers[0].storage_class.value = "ram" config.engine_params[0].storage.storage_tiers[0].scm_mount.value = "/mnt/daos" - config.engine_params[0].storage.storage_tiers[0].scm_size.value = 16 + config.engine_params[0].storage.storage_tiers[0].scm_size.value = 0 # Update the configuration file access points config.other_params.access_points.value = args.node_list.split(",") return create_config(args, config) diff --git a/src/tests/ftest/container/api_basic_attribute.py b/src/tests/ftest/container/api_basic_attribute.py index 3766114150a..fb6640cf7b9 100644 --- a/src/tests/ftest/container/api_basic_attribute.py +++ b/src/tests/ftest/container/api_basic_attribute.py @@ -5,10 +5,9 @@ ''' import traceback -from pydaos.raw import DaosApiError - from apricot import TestWithServers from general_utils import DaosTestError +from pydaos.raw import DaosApiError from test_utils_base import CallbackHandler diff --git a/src/tests/ftest/container/auto_oc_selection.py b/src/tests/ftest/container/auto_oc_selection.py index bf5e7a16380..cd2188e059e 100644 --- a/src/tests/ftest/container/auto_oc_selection.py +++ b/src/tests/ftest/container/auto_oc_selection.py @@ -3,9 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail class AutoOCSelectionTest(TestWithServers): diff --git a/src/tests/ftest/container/basic_snapshot.py b/src/tests/ftest/container/basic_snapshot.py index cc21c55ef57..f4204735583 100644 --- a/src/tests/ftest/container/basic_snapshot.py +++ b/src/tests/ftest/container/basic_snapshot.py @@ -3,10 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from pydaos.raw import DaosContainer, DaosSnapshot, DaosApiError - from apricot import TestWithServers from general_utils import get_random_bytes +from pydaos.raw import DaosApiError, DaosContainer, DaosSnapshot class BasicSnapshot(TestWithServers): diff --git a/src/tests/ftest/container/boundary.py b/src/tests/ftest/container/boundary.py index ad73b7d2bc6..670cab02593 100644 --- a/src/tests/ftest/container/boundary.py +++ b/src/tests/ftest/container/boundary.py @@ -4,13 +4,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import itertools import random - -from avocado.core.exceptions import TestFail +import time from apricot import TestWithServers +from avocado.core.exceptions import TestFail from general_utils import DaosTestError from thread_manager import ThreadManager diff --git a/src/tests/ftest/container/destroy.py b/src/tests/ftest/container/destroy.py index f5040835ed7..e93c4eac114 100644 --- a/src/tests/ftest/container/destroy.py +++ b/src/tests/ftest/container/destroy.py @@ -6,9 +6,8 @@ import traceback import uuid -from pydaos.raw import DaosApiError - from apricot import TestWithServers +from pydaos.raw import DaosApiError class ContainerDestroyTest(TestWithServers): diff --git a/src/tests/ftest/container/fill_destroy_loop.py b/src/tests/ftest/container/fill_destroy_loop.py index b71f88cc72e..ada553772b6 100644 --- a/src/tests/ftest/container/fill_destroy_loop.py +++ b/src/tests/ftest/container/fill_destroy_loop.py @@ -5,9 +5,8 @@ """ import os -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail from general_utils import bytes_to_human, human_to_bytes from run_utils import run_remote diff --git a/src/tests/ftest/container/full_pool_container_create.py b/src/tests/ftest/container/full_pool_container_create.py index e543ecae176..1874e78acde 100644 --- a/src/tests/ftest/container/full_pool_container_create.py +++ b/src/tests/ftest/container/full_pool_container_create.py @@ -4,8 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import time + from apricot import TestWithServers -from general_utils import get_random_bytes, DaosTestError +from general_utils import DaosTestError, get_random_bytes from test_utils_container import TestContainerData diff --git a/src/tests/ftest/container/label.py b/src/tests/ftest/container/label.py index 27894b18bcf..d9d8ddb6499 100644 --- a/src/tests/ftest/container/label.py +++ b/src/tests/ftest/container/label.py @@ -6,11 +6,10 @@ import string import uuid -from avocado.core.exceptions import TestFail - from apricot import TestWithServers -from general_utils import report_errors, get_random_string +from avocado.core.exceptions import TestFail from exception_utils import CommandFailure +from general_utils import get_random_string, report_errors class ContainerLabelTest(TestWithServers): diff --git a/src/tests/ftest/container/multiple_delete.py b/src/tests/ftest/container/multiple_delete.py index 4c54becd1d7..4abe99ec6af 100644 --- a/src/tests/ftest/container/multiple_delete.py +++ b/src/tests/ftest/container/multiple_delete.py @@ -5,8 +5,8 @@ """ import time -from ior_test_base import IorTestBase from general_utils import DaosTestError +from ior_test_base import IorTestBase SCM_THRESHOLD = 400000 diff --git a/src/tests/ftest/container/open.py b/src/tests/ftest/container/open.py index ba561ef08cb..aa20abf7348 100644 --- a/src/tests/ftest/container/open.py +++ b/src/tests/ftest/container/open.py @@ -6,9 +6,8 @@ import traceback import uuid -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail RESULT_PASS = "PASS" # nosec RESULT_FAIL = "FAIL" diff --git a/src/tests/ftest/container/query_attribute.py b/src/tests/ftest/container/query_attribute.py index 7a8cdd19501..60e137e94e6 100644 --- a/src/tests/ftest/container/query_attribute.py +++ b/src/tests/ftest/container/query_attribute.py @@ -4,10 +4,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import base64 + from apricot import TestWithServers from general_utils import report_errors - # Test container set-attr, get-attr, and list-attrs with different # types of characters. # pylint: disable=anomalous-backslash-in-string diff --git a/src/tests/ftest/container/query_properties.py b/src/tests/ftest/container/query_properties.py index 12cb84b36c0..92393a6d15f 100644 --- a/src/tests/ftest/container/query_properties.py +++ b/src/tests/ftest/container/query_properties.py @@ -5,9 +5,8 @@ ''' import ctypes -from pydaos.raw import daos_cref, DaosApiError, conversion, DaosContPropEnum - from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContPropEnum, conversion, daos_cref from test_utils_container import TestContainer diff --git a/src/tests/ftest/container/snapshot.py b/src/tests/ftest/container/snapshot.py index 4b7918f2f71..498d19f7b6f 100644 --- a/src/tests/ftest/container/snapshot.py +++ b/src/tests/ftest/container/snapshot.py @@ -5,10 +5,9 @@ """ import traceback -from pydaos.raw import DaosContainer, DaosSnapshot, DaosApiError, c_uuid_to_str - from apricot import TestWithServers from general_utils import get_random_bytes +from pydaos.raw import DaosApiError, DaosContainer, DaosSnapshot, c_uuid_to_str # pylint: disable=broad-except diff --git a/src/tests/ftest/control/config_generate_output.py b/src/tests/ftest/control/config_generate_output.py index d45f12fea01..a65790e9b73 100644 --- a/src/tests/ftest/control/config_generate_output.py +++ b/src/tests/ftest/control/config_generate_output.py @@ -4,11 +4,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' from collections import defaultdict -import yaml +import yaml from apricot import TestWithServers -from exception_utils import CommandFailure from dmg_utils import DmgCommand +from exception_utils import CommandFailure class ConfigGenerateOutput(TestWithServers): diff --git a/src/tests/ftest/control/config_generate_run.py b/src/tests/ftest/control/config_generate_run.py index ddc26966cf9..571e23c103e 100644 --- a/src/tests/ftest/control/config_generate_run.py +++ b/src/tests/ftest/control/config_generate_run.py @@ -5,7 +5,6 @@ ''' import yaml - from apricot import TestWithServers from server_utils import ServerFailed diff --git a/src/tests/ftest/control/daos_agent_config.py b/src/tests/ftest/control/daos_agent_config.py index fb5d6738b8b..d2f6c69ecc8 100644 --- a/src/tests/ftest/control/daos_agent_config.py +++ b/src/tests/ftest/control/daos_agent_config.py @@ -5,8 +5,8 @@ """ -from apricot import TestWithServers from agent_utils import include_local_host +from apricot import TestWithServers from exception_utils import CommandFailure diff --git a/src/tests/ftest/control/daos_server_helper.py b/src/tests/ftest/control/daos_server_helper.py index 44e6eec9270..dae932edcaf 100644 --- a/src/tests/ftest/control/daos_server_helper.py +++ b/src/tests/ftest/control/daos_server_helper.py @@ -3,9 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import getpass import os import stat -import getpass from apricot import TestWithServers from server_utils import ServerFailed diff --git a/src/tests/ftest/control/dmg_nvme_scan_test.py b/src/tests/ftest/control/dmg_nvme_scan_test.py index 04f3b3bcd02..25a41e14ec2 100644 --- a/src/tests/ftest/control/dmg_nvme_scan_test.py +++ b/src/tests/ftest/control/dmg_nvme_scan_test.py @@ -5,10 +5,9 @@ """ import os +from apricot import TestWithServers from avocado.utils import process - from dmg_utils import DmgCommand -from apricot import TestWithServers class DmgNvmeScanTest(TestWithServers): diff --git a/src/tests/ftest/control/dmg_pool_query_ranks.py b/src/tests/ftest/control/dmg_pool_query_ranks.py index d480f473f2e..d0cadef0754 100644 --- a/src/tests/ftest/control/dmg_pool_query_ranks.py +++ b/src/tests/ftest/control/dmg_pool_query_ranks.py @@ -5,8 +5,8 @@ """ import time -from exception_utils import CommandFailure from control_test_base import ControlTestBase +from exception_utils import CommandFailure class DmgPoolQueryRanks(ControlTestBase): diff --git a/src/tests/ftest/control/dmg_pool_query_test.py b/src/tests/ftest/control/dmg_pool_query_test.py index e71760a4d7b..200ea396cbf 100644 --- a/src/tests/ftest/control/dmg_pool_query_test.py +++ b/src/tests/ftest/control/dmg_pool_query_test.py @@ -5,8 +5,8 @@ """ from copy import deepcopy -from ior_test_base import IorTestBase from control_test_base import ControlTestBase +from ior_test_base import IorTestBase class DmgPoolQueryTest(ControlTestBase, IorTestBase): diff --git a/src/tests/ftest/control/dmg_storage_query.py b/src/tests/ftest/control/dmg_storage_query.py index 9021aeea0bc..f1d8b76bef0 100644 --- a/src/tests/ftest/control/dmg_storage_query.py +++ b/src/tests/ftest/control/dmg_storage_query.py @@ -7,11 +7,10 @@ import re import avocado - from control_test_base import ControlTestBase -from dmg_utils import get_storage_query_pool_info, get_storage_query_device_info +from dmg_utils import get_storage_query_device_info, get_storage_query_pool_info from exception_utils import CommandFailure -from general_utils import list_to_str, dict_to_str +from general_utils import dict_to_str, list_to_str class DmgStorageQuery(ControlTestBase): diff --git a/src/tests/ftest/control/dmg_storage_scan_scm.py b/src/tests/ftest/control/dmg_storage_scan_scm.py index 8f69f9f0edc..4c3373c0d0f 100644 --- a/src/tests/ftest/control/dmg_storage_scan_scm.py +++ b/src/tests/ftest/control/dmg_storage_scan_scm.py @@ -5,8 +5,8 @@ """ import os -from general_utils import pcmd, run_pcmd from control_test_base import ControlTestBase +from general_utils import pcmd, run_pcmd class DmgStorageScanSCMTest(ControlTestBase): diff --git a/src/tests/ftest/control/dmg_system_cleanup.py b/src/tests/ftest/control/dmg_system_cleanup.py index de72667d0c5..9051dd28dab 100644 --- a/src/tests/ftest/control/dmg_system_cleanup.py +++ b/src/tests/ftest/control/dmg_system_cleanup.py @@ -4,9 +4,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from socket import gethostname + +from apricot import TestWithServers from avocado.core.exceptions import TestFail from pydaos.raw import DaosPool -from apricot import TestWithServers class DmgSystemCleanupTest(TestWithServers): diff --git a/src/tests/ftest/control/dmg_system_reformat.py b/src/tests/ftest/control/dmg_system_reformat.py index bdf5d583052..d20354018c8 100644 --- a/src/tests/ftest/control/dmg_system_reformat.py +++ b/src/tests/ftest/control/dmg_system_reformat.py @@ -5,9 +5,8 @@ """ import time -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail from exception_utils import CommandFailure from test_utils_pool import add_pool, get_size_params diff --git a/src/tests/ftest/control/dmg_telemetry_io_latency.py b/src/tests/ftest/control/dmg_telemetry_io_latency.py index 034389270be..82ad47eb2fb 100644 --- a/src/tests/ftest/control/dmg_telemetry_io_latency.py +++ b/src/tests/ftest/control/dmg_telemetry_io_latency.py @@ -4,11 +4,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from general_utils import report_errors from ior_test_base import IorTestBase +from oclass_utils import extract_redundancy_factor from telemetry_test_base import TestWithTelemetry from telemetry_utils import TelemetryUtils -from oclass_utils import extract_redundancy_factor -from general_utils import report_errors def convert_to_number(size): diff --git a/src/tests/ftest/control/dmg_telemetry_nvme.py b/src/tests/ftest/control/dmg_telemetry_nvme.py index d6cdc26197a..0e211d8d617 100644 --- a/src/tests/ftest/control/dmg_telemetry_nvme.py +++ b/src/tests/ftest/control/dmg_telemetry_nvme.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from telemetry_test_base import TestWithTelemetry from apricot import TestWithServers +from telemetry_test_base import TestWithTelemetry from telemetry_utils import TelemetryUtils diff --git a/src/tests/ftest/control/log_entry.py b/src/tests/ftest/control/log_entry.py index 641b49d8e97..c932f775765 100644 --- a/src/tests/ftest/control/log_entry.py +++ b/src/tests/ftest/control/log_entry.py @@ -6,9 +6,8 @@ import contextlib import re -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers +from ClusterShell.NodeSet import NodeSet from general_utils import get_journalctl, journalctl_time, wait_for_result from run_utils import run_remote diff --git a/src/tests/ftest/control/ms_failover.py b/src/tests/ftest/control/ms_failover.py index 54fc7d90b3a..f12c0fe27e5 100644 --- a/src/tests/ftest/control/ms_failover.py +++ b/src/tests/ftest/control/ms_failover.py @@ -6,9 +6,8 @@ import socket import time -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers +from ClusterShell.NodeSet import NodeSet class ManagementServiceFailover(TestWithServers): diff --git a/src/tests/ftest/control/ms_resilience.py b/src/tests/ftest/control/ms_resilience.py index e0ca68ed575..4409c7beff6 100644 --- a/src/tests/ftest/control/ms_resilience.py +++ b/src/tests/ftest/control/ms_resilience.py @@ -6,9 +6,8 @@ import socket import time -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers +from ClusterShell.NodeSet import NodeSet from run_utils import stop_processes diff --git a/src/tests/ftest/control/ssd_socket.py b/src/tests/ftest/control/ssd_socket.py index acebfdeab0f..2a899dfb446 100644 --- a/src/tests/ftest/control/ssd_socket.py +++ b/src/tests/ftest/control/ssd_socket.py @@ -5,8 +5,9 @@ """ import os from textwrap import wrap -from general_utils import pcmd, run_pcmd + from control_test_base import ControlTestBase +from general_utils import pcmd, run_pcmd class SSDSocketTest(ControlTestBase): diff --git a/src/tests/ftest/control/super_block_versioning.py b/src/tests/ftest/control/super_block_versioning.py index 3b8576caf79..5a9141c5f95 100644 --- a/src/tests/ftest/control/super_block_versioning.py +++ b/src/tests/ftest/control/super_block_versioning.py @@ -7,8 +7,8 @@ import os -from general_utils import check_file_exists, pcmd from apricot import TestWithServers +from general_utils import check_file_exists, pcmd class SuperBlockVersioning(TestWithServers): diff --git a/src/tests/ftest/control/version.py b/src/tests/ftest/control/version.py index 93a5110c490..247b7072953 100644 --- a/src/tests/ftest/control/version.py +++ b/src/tests/ftest/control/version.py @@ -3,11 +3,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import re import json +import re from apricot import TestWithServers -from general_utils import run_pcmd, report_errors, append_error +from general_utils import append_error, report_errors, run_pcmd from server_utils_base import DaosServerCommandRunner diff --git a/src/tests/ftest/daos_racer/parallel.py b/src/tests/ftest/daos_racer/parallel.py index bf3116891f9..6dba46ef7d2 100755 --- a/src/tests/ftest/daos_racer/parallel.py +++ b/src/tests/ftest/daos_racer/parallel.py @@ -6,8 +6,8 @@ """ from apricot import TestWithServers -from exception_utils import CommandFailure from daos_racer_utils import DaosRacerCommand +from exception_utils import CommandFailure class DaosRacerParallelTest(TestWithServers): diff --git a/src/tests/ftest/daos_test/dfs.py b/src/tests/ftest/daos_test/dfs.py index 04eb6073666..c5d5d4adaef 100644 --- a/src/tests/ftest/daos_test/dfs.py +++ b/src/tests/ftest/daos_test/dfs.py @@ -5,6 +5,7 @@ """ import os + from daos_core_base import DaosCoreBase diff --git a/src/tests/ftest/daos_test/suite.single_host.yaml b/src/tests/ftest/daos_test/suite.single_host.yaml new file mode 100644 index 00000000000..e4fba4a3e12 --- /dev/null +++ b/src/tests/ftest/daos_test/suite.single_host.yaml @@ -0,0 +1,159 @@ +# change host names to your reserved nodes, the +# required quantity is indicated by the placeholders +hosts: + test_servers: 1 +# Note that subtests below can set their own timeout so this +# should be a general average of all tests not including outliers +# (I'm looking at you "rebuild tests") +timeout: 600 +timeouts: + test_daos_degraded_mode: 450 + test_daos_management: 110 + test_daos_pool: 180 + test_daos_container: 450 + test_daos_epoch: 125 + test_daos_verify_consistency: 105 + test_daos_io: 290 + test_daos_ec_io: 450 + test_daos_ec_obj: 600 + test_daos_object_array: 105 + test_daos_array: 106 + test_daos_kv: 105 + test_daos_capability: 104 + test_daos_epoch_recovery: 104 + test_daos_md_replication: 104 + test_daos_rebuild_simple: 1800 + test_daos_drain_simple: 3600 + test_daos_extend_simple: 3600 + test_daos_oid_allocator: 640 + test_daos_checksum: 500 + test_daos_rebuild_ec: 6400 + test_daos_aggregate_ec: 200 + test_daos_degraded_ec: 1900 + test_daos_dedup: 220 + test_daos_upgrade: 300 + test_daos_pipeline: 60 +pool: + size: 16G +server_config: + name: daos_server + engines_per_host: 1 + engines: + 0: + storage: auto + transport_config: + allow_insecure: true + system_ram_reserved: 64 +agent_config: + transport_config: + allow_insecure: true +dmg: + transport_config: + allow_insecure: true +daos_tests: + num_clients: + test_daos_degraded_mode: 1 + test_daos_management: 1 + test_daos_pool: 2 + test_daos_container: 1 + test_daos_epoch: 1 + test_daos_single_rdg_tx: 1 + test_daos_distributed_tx: 1 + test_daos_verify_consistency: 1 + test_daos_io: 1 + test_daos_ec_io: 1 + test_daos_ec_obj: 1 + test_daos_object_array: 1 + test_daos_array: 1 + test_daos_kv: 1 + test_daos_capability: 2 + test_daos_epoch_recovery: 1 + test_daos_md_replication: 2 + test_daos_rebuild_simple: 1 + test_daos_drain_simple: 1 + test_daos_extend_simple: 1 + test_daos_oid_allocator: 1 + test_daos_checksum: 1 + test_daos_rebuild_ec: 1 + test_daos_aggregate_ec: 1 + test_daos_degraded_ec: 1 + test_daos_dedup: 1 + test_daos_upgrade: 1 + test_daos_pipeline: 1 + test_name: + test_daos_degraded_mode: DAOS_Degraded_Mode + test_daos_management: DAOS_Management + test_daos_pool: DAOS_Pool + test_daos_container: DAOS_Container + test_daos_epoch: DAOS_Epoch + test_daos_single_rdg_tx: DAOS_Single_RDG_TX + test_daos_distributed_tx: DAOS_Distributed_TX + test_daos_verify_consistency: DAOS_Verify_Consistency + test_daos_io: DAOS_IO + test_daos_ec_io: DAOS_IO_EC_4P2G1 + test_daos_ec_obj: DAOS_EC + test_daos_object_array: DAOS_Object_Array + test_daos_array: DAOS_Array + test_daos_kv: DAOS_KV + test_daos_capability: DAOS_Capability + test_daos_epoch_recovery: DAOS_Epoch_Recovery + test_daos_md_replication: DAOS_MD_Replication + test_daos_rebuild_simple: DAOS_Rebuild_Simple + test_daos_drain_simple: DAOS_Drain_Simple + test_daos_oid_allocator: DAOS_OID_Allocator + test_daos_checksum: DAOS_Checksum + test_daos_rebuild_ec: DAOS_Rebuild_EC + test_daos_aggregate_ec: DAOS_Aggregate_EC + test_daos_degraded_ec: DAOS_Degraded_EC + test_daos_dedup: DAOS_Dedup + test_daos_extend_simple: DAOS_Extend_Simple + test_daos_upgrade: DAOS_Upgrade + test_daos_pipeline: DAOS_Pipeline + daos_test: + test_daos_degraded_mode: d + test_daos_management: m + test_daos_pool: p + test_daos_container: c + test_daos_epoch: e + test_daos_single_rdg_tx: t + test_daos_distributed_tx: T + test_daos_verify_consistency: V + test_daos_io: i + test_daos_ec_io: i + test_daos_ec_obj: I + test_daos_object_array: A + test_daos_array: D + test_daos_kv: K + test_daos_capability: C + test_daos_epoch_recovery: o + test_daos_md_replication: R + test_daos_rebuild_simple: v + test_daos_drain_simple: b + test_daos_extend_simple: B + test_daos_oid_allocator: O + test_daos_checksum: z + test_daos_rebuild_ec: S + test_daos_aggregate_ec: Z + test_daos_degraded_ec: X + test_daos_dedup: U + test_daos_upgrade: G + test_daos_pipeline: P + args: + test_daos_ec_io: -l"EC_4P2G1" + test_daos_rebuild_ec: -s5 + test_daos_md_replication: -s5 + scalable_endpoint: + test_daos_degraded_mode: true + stopped_ranks: + test_daos_degraded_mode: [5, 6, 7] + test_daos_oid_allocator: [6, 7] + pools_created: + test_daos_management: 4 + test_daos_pool: 9 + test_daos_container: 17 + test_daos_distributed_tx: 5 + test_daos_rebuild_simple: 21 + test_daos_drain_simple: 8 + test_daos_extend_simple: 5 + test_daos_rebuild_ec: 43 + test_daos_degraded_ec: 29 diff --git a/src/tests/ftest/daos_test/suite.yaml b/src/tests/ftest/daos_test/suite.yaml index 279950ca2a2..4da1d5e2a64 100644 --- a/src/tests/ftest/daos_test/suite.yaml +++ b/src/tests/ftest/daos_test/suite.yaml @@ -15,7 +15,7 @@ timeouts: test_daos_verify_consistency: 105 test_daos_io: 290 test_daos_ec_io: 450 - test_daos_ec_obj: 600 + test_daos_ec_obj: 700 test_daos_object_array: 105 test_daos_array: 106 test_daos_kv: 105 diff --git a/src/tests/ftest/daos_vol/bigio.py b/src/tests/ftest/daos_vol/bigio.py index 54bbf925471..75094dc3537 100644 --- a/src/tests/ftest/daos_vol/bigio.py +++ b/src/tests/ftest/daos_vol/bigio.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from vol_test_base import VolTestBase from job_manager_utils import get_job_manager +from vol_test_base import VolTestBase class DaosVolBigIO(VolTestBase): diff --git a/src/tests/ftest/daos_vol/h5_suite.py b/src/tests/ftest/daos_vol/h5_suite.py index a1eba76677b..fd4d339fb86 100644 --- a/src/tests/ftest/daos_vol/h5_suite.py +++ b/src/tests/ftest/daos_vol/h5_suite.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from vol_test_base import VolTestBase from job_manager_utils import get_job_manager +from vol_test_base import VolTestBase class DaosVol(VolTestBase): diff --git a/src/tests/ftest/datamover/copy_procs.py b/src/tests/ftest/datamover/copy_procs.py index 4119fa723ae..ce980f373f4 100644 --- a/src/tests/ftest/datamover/copy_procs.py +++ b/src/tests/ftest/datamover/copy_procs.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' from os.path import join + from data_mover_test_base import DataMoverTestBase diff --git a/src/tests/ftest/datamover/dst_create.py b/src/tests/ftest/datamover/dst_create.py index e06c46d129c..b2956045e4b 100644 --- a/src/tests/ftest/datamover/dst_create.py +++ b/src/tests/ftest/datamover/dst_create.py @@ -6,10 +6,9 @@ from os.path import join import avocado -from pydaos.raw import DaosApiError - from data_mover_test_base import DataMoverTestBase from duns_utils import format_path +from pydaos.raw import DaosApiError class DmvrDstCreate(DataMoverTestBase): diff --git a/src/tests/ftest/datamover/large_dir.py b/src/tests/ftest/datamover/large_dir.py index 720e95f458d..17751a3d5f4 100644 --- a/src/tests/ftest/datamover/large_dir.py +++ b/src/tests/ftest/datamover/large_dir.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' import os + from data_mover_test_base import DataMoverTestBase from duns_utils import format_path diff --git a/src/tests/ftest/datamover/large_file.py b/src/tests/ftest/datamover/large_file.py index 355ec5eecc1..6fc9faf03e0 100644 --- a/src/tests/ftest/datamover/large_file.py +++ b/src/tests/ftest/datamover/large_file.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' import os + from data_mover_test_base import DataMoverTestBase diff --git a/src/tests/ftest/datamover/negative.py b/src/tests/ftest/datamover/negative.py index 32142ffe654..5d9b36adc60 100644 --- a/src/tests/ftest/datamover/negative.py +++ b/src/tests/ftest/datamover/negative.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from os.path import join import uuid +from os.path import join from data_mover_test_base import DataMoverTestBase from duns_utils import format_path diff --git a/src/tests/ftest/datamover/negative_space.py b/src/tests/ftest/datamover/negative_space.py index 0c9af6cd48b..69d7fd7ce5b 100644 --- a/src/tests/ftest/datamover/negative_space.py +++ b/src/tests/ftest/datamover/negative_space.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' from os.path import join + from data_mover_test_base import DataMoverTestBase diff --git a/src/tests/ftest/datamover/obj_small.py b/src/tests/ftest/datamover/obj_small.py index 80812015198..00fae2a6257 100644 --- a/src/tests/ftest/datamover/obj_small.py +++ b/src/tests/ftest/datamover/obj_small.py @@ -3,10 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from pydaos.raw import DaosApiError import avocado - from data_mover_test_base import DataMoverTestBase +from pydaos.raw import DaosApiError class DmvrObjSmallTest(DataMoverTestBase): diff --git a/src/tests/ftest/datamover/posix_meta_entry.py b/src/tests/ftest/datamover/posix_meta_entry.py index a39b404ac1c..40268fc7485 100644 --- a/src/tests/ftest/datamover/posix_meta_entry.py +++ b/src/tests/ftest/datamover/posix_meta_entry.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from os.path import join + from data_mover_test_base import DataMoverTestBase from exception_utils import CommandFailure diff --git a/src/tests/ftest/datamover/posix_preserve_props.py b/src/tests/ftest/datamover/posix_preserve_props.py index c05e609a4f3..07fa6bd66c0 100644 --- a/src/tests/ftest/datamover/posix_preserve_props.py +++ b/src/tests/ftest/datamover/posix_preserve_props.py @@ -5,11 +5,10 @@ ''' from os.path import join -from pydaos.raw import DaosApiError import avocado - from data_mover_test_base import DataMoverTestBase from duns_utils import format_path +from pydaos.raw import DaosApiError class DmvrPreserveProps(DataMoverTestBase): diff --git a/src/tests/ftest/datamover/posix_subsets.py b/src/tests/ftest/datamover/posix_subsets.py index 791156307cb..ad62672d096 100644 --- a/src/tests/ftest/datamover/posix_subsets.py +++ b/src/tests/ftest/datamover/posix_subsets.py @@ -3,8 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from os.path import join import re +from os.path import join + from data_mover_test_base import DataMoverTestBase diff --git a/src/tests/ftest/datamover/posix_symlinks.py b/src/tests/ftest/datamover/posix_symlinks.py index ae938783870..cdffdd9085e 100644 --- a/src/tests/ftest/datamover/posix_symlinks.py +++ b/src/tests/ftest/datamover/posix_symlinks.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' from os.path import join + from data_mover_test_base import DataMoverTestBase diff --git a/src/tests/ftest/datamover/serial_small.py b/src/tests/ftest/datamover/serial_small.py index afae5e76451..9d1b6243d3f 100644 --- a/src/tests/ftest/datamover/serial_small.py +++ b/src/tests/ftest/datamover/serial_small.py @@ -3,10 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from pydaos.raw import DaosApiError import avocado - from data_mover_test_base import DataMoverTestBase +from pydaos.raw import DaosApiError class DmvrSerialSmall(DataMoverTestBase): diff --git a/src/tests/ftest/dbench/dbench.py b/src/tests/ftest/dbench/dbench.py index 907bcc51d18..05081fb4039 100644 --- a/src/tests/ftest/dbench/dbench.py +++ b/src/tests/ftest/dbench/dbench.py @@ -5,9 +5,9 @@ """ from ClusterShell.NodeSet import NodeSet +from dbench_utils import Dbench from dfuse_test_base import DfuseTestBase from exception_utils import CommandFailure -from dbench_utils import Dbench class DbenchTest(DfuseTestBase): diff --git a/src/tests/ftest/deployment/agent_failure.py b/src/tests/ftest/deployment/agent_failure.py index 4cefa79262e..181b43b689c 100644 --- a/src/tests/ftest/deployment/agent_failure.py +++ b/src/tests/ftest/deployment/agent_failure.py @@ -3,16 +3,15 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import os import threading +import time from ClusterShell.NodeSet import NodeSet - +from command_utils_base import CommandFailure +from general_utils import get_journalctl, journalctl_time, report_errors from ior_test_base import IorTestBase from ior_utils import IorCommand -from general_utils import report_errors, get_journalctl, journalctl_time -from command_utils_base import CommandFailure from job_manager_utils import get_job_manager from run_utils import stop_processes diff --git a/src/tests/ftest/deployment/basic_checkout.py b/src/tests/ftest/deployment/basic_checkout.py index 3a700f27223..52a828e8329 100644 --- a/src/tests/ftest/deployment/basic_checkout.py +++ b/src/tests/ftest/deployment/basic_checkout.py @@ -4,9 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from performance_test_base import PerformanceTestBase from data_mover_test_base import DataMoverTestBase from exception_utils import CommandFailure +from performance_test_base import PerformanceTestBase class BasicCheckout(PerformanceTestBase): diff --git a/src/tests/ftest/deployment/critical_integration.py b/src/tests/ftest/deployment/critical_integration.py index a714ed28daf..8afdf18cfcc 100644 --- a/src/tests/ftest/deployment/critical_integration.py +++ b/src/tests/ftest/deployment/critical_integration.py @@ -6,16 +6,12 @@ import json +# Imports need to be split or python fails to import +from apricot import TestWithoutServers, TestWithServers from ClusterShell.NodeSet import NodeSet - -from general_utils import run_command, DaosTestError, get_journalctl, journalctl_time -from ior_test_base import IorTestBase from exception_utils import CommandFailure - -# Imports need to be split or python fails to import -from apricot import TestWithServers -from apricot import TestWithoutServers - +from general_utils import DaosTestError, get_journalctl, journalctl_time, run_command +from ior_test_base import IorTestBase # pylint: disable-next=fixme # TODO Provision all daos nodes using provisioning tool provided by HPCM diff --git a/src/tests/ftest/deployment/disk_failure.py b/src/tests/ftest/deployment/disk_failure.py index 79c37ef3cf7..23f2132171c 100644 --- a/src/tests/ftest/deployment/disk_failure.py +++ b/src/tests/ftest/deployment/disk_failure.py @@ -9,12 +9,11 @@ from avocado import fail_on from ClusterShell.NodeSet import NodeSet - -from dmg_utils import get_storage_query_device_info, get_dmg_response +from dmg_utils import get_dmg_response, get_storage_query_device_info from exception_utils import CommandFailure from general_utils import list_to_str -from test_utils_pool import add_pool from osa_utils import OSAUtils +from test_utils_pool import add_pool class DiskFailureTest(OSAUtils): diff --git a/src/tests/ftest/deployment/io_sys_admin.py b/src/tests/ftest/deployment/io_sys_admin.py index 0c85d01ea5a..2eaaf8be324 100644 --- a/src/tests/ftest/deployment/io_sys_admin.py +++ b/src/tests/ftest/deployment/io_sys_admin.py @@ -6,10 +6,10 @@ import time -from file_count_test_base import FileCountTestBase +import security_test_base as secTestBase from data_mover_test_base import DataMoverTestBase +from file_count_test_base import FileCountTestBase from general_utils import human_to_bytes -import security_test_base as secTestBase from test_utils_pool import check_pool_creation diff --git a/src/tests/ftest/deployment/ior_per_rank.py b/src/tests/ftest/deployment/ior_per_rank.py index 86f53978a31..f914216f326 100644 --- a/src/tests/ftest/deployment/ior_per_rank.py +++ b/src/tests/ftest/deployment/ior_per_rank.py @@ -5,10 +5,9 @@ """ from avocado.core.exceptions import TestFail - +from general_utils import DaosTestError, percent_change from ior_test_base import IorTestBase from ior_utils import IorCommand, IorMetrics -from general_utils import percent_change, DaosTestError class IorPerRank(IorTestBase): diff --git a/src/tests/ftest/deployment/network_failure.py b/src/tests/ftest/deployment/network_failure.py index b52a6c5bf88..8c3de5cd414 100644 --- a/src/tests/ftest/deployment/network_failure.py +++ b/src/tests/ftest/deployment/network_failure.py @@ -6,15 +6,15 @@ import os import time from collections import defaultdict -from ClusterShell.NodeSet import NodeSet +from ClusterShell.NodeSet import NodeSet +from command_utils_base import CommandFailure +from dmg_utils import check_system_query_status +from general_utils import report_errors, run_pcmd from ior_test_base import IorTestBase from ior_utils import IorCommand -from general_utils import report_errors, run_pcmd -from command_utils_base import CommandFailure from job_manager_utils import get_job_manager from network_utils import update_network_interface -from dmg_utils import check_system_query_status class NetworkFailureTest(IorTestBase): diff --git a/src/tests/ftest/deployment/server_rank_failure.py b/src/tests/ftest/deployment/server_rank_failure.py index 8392d4283d7..51d6c2e660f 100644 --- a/src/tests/ftest/deployment/server_rank_failure.py +++ b/src/tests/ftest/deployment/server_rank_failure.py @@ -3,16 +3,15 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import os import threading +import time from ClusterShell.NodeSet import NodeSet - +from command_utils_base import CommandFailure +from general_utils import report_errors from ior_test_base import IorTestBase from ior_utils import IorCommand -from general_utils import report_errors -from command_utils_base import CommandFailure from job_manager_utils import get_job_manager from run_utils import stop_processes diff --git a/src/tests/ftest/deployment/target_failure.py b/src/tests/ftest/deployment/target_failure.py index b54b9e02549..2ddeebfccc9 100644 --- a/src/tests/ftest/deployment/target_failure.py +++ b/src/tests/ftest/deployment/target_failure.py @@ -3,14 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import os import threading +import time +from command_utils_base import CommandFailure +from general_utils import report_errors from ior_test_base import IorTestBase from ior_utils import IorCommand -from general_utils import report_errors -from command_utils_base import CommandFailure from job_manager_utils import get_job_manager diff --git a/src/tests/ftest/dfuse/bash.py b/src/tests/ftest/dfuse/bash.py index 4f2b2aa165b..db1b3bbfe24 100644 --- a/src/tests/ftest/dfuse/bash.py +++ b/src/tests/ftest/dfuse/bash.py @@ -4,9 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os -from ClusterShell.NodeSet import NodeSet import general_utils +from ClusterShell.NodeSet import NodeSet from dfuse_test_base import DfuseTestBase from exception_utils import CommandFailure diff --git a/src/tests/ftest/dfuse/caching_check.py b/src/tests/ftest/dfuse/caching_check.py index 8806b9d588d..852a24f0dfd 100644 --- a/src/tests/ftest/dfuse/caching_check.py +++ b/src/tests/ftest/dfuse/caching_check.py @@ -4,9 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from general_utils import percent_change from ior_test_base import IorTestBase from ior_utils import IorCommand, IorMetrics -from general_utils import percent_change class DfuseCachingCheck(IorTestBase): @@ -43,34 +43,34 @@ def test_dfuse_caching_check(self): # update flag self.ior_cmd.update_params(flags=flags[0]) - # run ior to write to the dfuse mount point + self.log_step('Write to the dfuse mount point') self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False) - # update ior flag to read + self.log_step('Get baseline read performance from dfuse with caching disabled') self.ior_cmd.update_params(flags=flags[1]) - # run ior to read and store the read performance base_read_arr = [] out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False) base_read_arr.append(IorCommand.get_ior_metrics(out)) - # run ior again to read with caching disabled and store performance out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False) base_read_arr.append(IorCommand.get_ior_metrics(out)) # the index of max_mib max_mib = IorMetrics.MAX_MIB - # unmount dfuse and mount again with caching enabled + self.log_step('Re-mount dfuse with caching enabled') self.dfuse.unmount(tries=1) self.dfuse.update_params(disable_caching=False) self.dfuse.run() - # run ior to obtain first read performance after mount + + self.log_step('Get first read performance with caching enabled') out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False) base_read_arr.append(IorCommand.get_ior_metrics(out)) - # run ior again to obtain second read performance with caching enabled - # second read should be multiple times greater than first read + + self.log_step('Get cached read performance') out = self.run_ior_with_pool(fail_on_warning=False) with_caching = IorCommand.get_ior_metrics(out) - # verify cached read performance is multiple times greater than without caching + + self.log_step('Verify cached read performance is greater than first read') # Log all the values first, then do the assert so that failures can be checked easily. for base_read in base_read_arr: actual_change = percent_change(base_read[0][max_mib], with_caching[0][max_mib]) diff --git a/src/tests/ftest/dfuse/caching_check.yaml b/src/tests/ftest/dfuse/caching_check.yaml index 66928e7de04..304bc0ae77e 100644 --- a/src/tests/ftest/dfuse/caching_check.yaml +++ b/src/tests/ftest/dfuse/caching_check.yaml @@ -1,7 +1,7 @@ hosts: test_servers: 3 test_clients: 1 -timeout: 240 +timeout: 300 server_config: name: daos_server engines_per_host: 1 @@ -11,7 +11,6 @@ server_config: storage: auto pool: size: 50% - control_method: dmg container: type: POSIX control_method: daos diff --git a/src/tests/ftest/dfuse/container_type.py b/src/tests/ftest/dfuse/container_type.py index 0f93a616beb..56ea7a25a08 100644 --- a/src/tests/ftest/dfuse/container_type.py +++ b/src/tests/ftest/dfuse/container_type.py @@ -4,7 +4,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from avocado.core.exceptions import TestFail - from dfuse_test_base import DfuseTestBase diff --git a/src/tests/ftest/dfuse/enospace.py b/src/tests/ftest/dfuse/enospace.py index 5ef5a4cba19..4f4c1136baa 100644 --- a/src/tests/ftest/dfuse/enospace.py +++ b/src/tests/ftest/dfuse/enospace.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import os import errno +import os from dfuse_test_base import DfuseTestBase diff --git a/src/tests/ftest/dfuse/find.py b/src/tests/ftest/dfuse/find.py index 180adc7a89b..f583e303816 100644 --- a/src/tests/ftest/dfuse/find.py +++ b/src/tests/ftest/dfuse/find.py @@ -4,12 +4,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os -import sys -import string import random -from ClusterShell.NodeSet import NodeSet +import string +import sys import general_utils +from ClusterShell.NodeSet import NodeSet from dfuse_test_base import DfuseTestBase from dfuse_utils import get_dfuse, start_dfuse from exception_utils import CommandFailure diff --git a/src/tests/ftest/dfuse/mu_mount.py b/src/tests/ftest/dfuse/mu_mount.py index 8cf47b8c9b4..8083cb684db 100644 --- a/src/tests/ftest/dfuse/mu_mount.py +++ b/src/tests/ftest/dfuse/mu_mount.py @@ -7,9 +7,8 @@ from getpass import getuser from ClusterShell.NodeSet import NodeSet - -from run_utils import run_remote, command_as_user from dfuse_test_base import DfuseTestBase +from run_utils import command_as_user, run_remote class DfuseMUMount(DfuseTestBase): diff --git a/src/tests/ftest/dfuse/mu_perms.py b/src/tests/ftest/dfuse/mu_perms.py index 9d571972bc9..f9cbe4450d8 100644 --- a/src/tests/ftest/dfuse/mu_perms.py +++ b/src/tests/ftest/dfuse/mu_perms.py @@ -3,16 +3,15 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os -from itertools import product -import time import re +import time +from itertools import product from ClusterShell.NodeSet import NodeSet - from dfuse_test_base import DfuseTestBase -from dfuse_utils import get_dfuse, start_dfuse, VerifyPermsCommand +from dfuse_utils import VerifyPermsCommand, get_dfuse, start_dfuse +from run_utils import command_as_user, run_remote from user_utils import get_chown_command -from run_utils import run_remote, command_as_user class DfuseMUPerms(DfuseTestBase): diff --git a/src/tests/ftest/dfuse/posix_stat.py b/src/tests/ftest/dfuse/posix_stat.py index e8dc75c3bee..57030f6567a 100644 --- a/src/tests/ftest/dfuse/posix_stat.py +++ b/src/tests/ftest/dfuse/posix_stat.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ior_test_base import IorTestBase from general_utils import get_remote_file_size, run_pcmd +from ior_test_base import IorTestBase class POSIXStatTest(IorTestBase): diff --git a/src/tests/ftest/dfuse/simul.py b/src/tests/ftest/dfuse/simul.py index e8b5d920ba5..aef80b5d651 100644 --- a/src/tests/ftest/dfuse/simul.py +++ b/src/tests/ftest/dfuse/simul.py @@ -5,10 +5,11 @@ """ import os + from avocado import fail_on -from exception_utils import MPILoadError from dfuse_test_base import DfuseTestBase from env_modules import load_mpi +from exception_utils import MPILoadError from general_utils import DaosTestError, run_command diff --git a/src/tests/ftest/dfuse/sparse_file.py b/src/tests/ftest/dfuse/sparse_file.py index 70e6460fd67..6b8521614b1 100644 --- a/src/tests/ftest/dfuse/sparse_file.py +++ b/src/tests/ftest/dfuse/sparse_file.py @@ -4,10 +4,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from getpass import getuser import os -import paramiko +from getpass import getuser +import paramiko from general_utils import get_remote_file_size from ior_test_base import IorTestBase diff --git a/src/tests/ftest/dtx/basic.py b/src/tests/ftest/dtx/basic.py index 6c27cbd6377..96ecc16049c 100644 --- a/src/tests/ftest/dtx/basic.py +++ b/src/tests/ftest/dtx/basic.py @@ -6,9 +6,8 @@ import time import traceback -from pydaos.raw import DaosContainer, DaosApiError, c_uuid_to_str - from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContainer, c_uuid_to_str class BasicTxTest(TestWithServers): diff --git a/src/tests/ftest/erasurecode/offline_rebuild_aggregation.py b/src/tests/ftest/erasurecode/offline_rebuild_aggregation.py index 7b5ede44aaf..5dc238f7a2e 100644 --- a/src/tests/ftest/erasurecode/offline_rebuild_aggregation.py +++ b/src/tests/ftest/erasurecode/offline_rebuild_aggregation.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' import time + from ec_utils import ErasureCodeIor diff --git a/src/tests/ftest/erasurecode/rebuild_disabled.py b/src/tests/ftest/erasurecode/rebuild_disabled.py index bdbecd288a7..7310dcee731 100644 --- a/src/tests/ftest/erasurecode/rebuild_disabled.py +++ b/src/tests/ftest/erasurecode/rebuild_disabled.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' import time + from ec_utils import ErasureCodeIor diff --git a/src/tests/ftest/erasurecode/rebuild_disabled_single.py b/src/tests/ftest/erasurecode/rebuild_disabled_single.py index 1c6fe2dfe5f..ff8f77d3165 100644 --- a/src/tests/ftest/erasurecode/rebuild_disabled_single.py +++ b/src/tests/ftest/erasurecode/rebuild_disabled_single.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' import time + from ec_utils import ErasureCodeSingle diff --git a/src/tests/ftest/erasurecode/space_usage.py b/src/tests/ftest/erasurecode/space_usage.py index 9abff06443e..7fa40365e76 100644 --- a/src/tests/ftest/erasurecode/space_usage.py +++ b/src/tests/ftest/erasurecode/space_usage.py @@ -3,9 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ior_test_base import IorTestBase -from data_utils import list_stats, dict_subtract, dict_extract_values +from data_utils import dict_extract_values, dict_subtract, list_stats from general_utils import percent_change +from ior_test_base import IorTestBase from oclass_utils import calculate_ec_targets_used diff --git a/src/tests/ftest/erasurecode/truncate.py b/src/tests/ftest/erasurecode/truncate.py index 3218b5c86b9..dc8ff91f67a 100644 --- a/src/tests/ftest/erasurecode/truncate.py +++ b/src/tests/ftest/erasurecode/truncate.py @@ -6,7 +6,7 @@ import os from fio_test_base import FioBase -from general_utils import run_pcmd, get_remote_file_size +from general_utils import get_remote_file_size, run_pcmd class Ecodtruncate(FioBase): diff --git a/src/tests/ftest/fault_domain/fault_domain.py b/src/tests/ftest/fault_domain/fault_domain.py index 431d055db56..c4e893383f7 100644 --- a/src/tests/ftest/fault_domain/fault_domain.py +++ b/src/tests/ftest/fault_domain/fault_domain.py @@ -3,9 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers, skipForTicket +from ClusterShell.NodeSet import NodeSet class FaultDomain(TestWithServers): diff --git a/src/tests/ftest/fault_injection/ec.py b/src/tests/ftest/fault_injection/ec.py index 7e3411d0eb8..873ddd1741c 100644 --- a/src/tests/ftest/fault_injection/ec.py +++ b/src/tests/ftest/fault_injection/ec.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from ior_test_base import IorTestBase from fio_test_base import FioBase +from ior_test_base import IorTestBase class EcodFaultInjection(IorTestBase, FioBase): diff --git a/src/tests/ftest/fault_injection/pool.py b/src/tests/ftest/fault_injection/pool.py index 05aae07d543..907772d7f36 100644 --- a/src/tests/ftest/fault_injection/pool.py +++ b/src/tests/ftest/fault_injection/pool.py @@ -5,6 +5,7 @@ """ from random import randint + from apricot import TestWithServers from exception_utils import CommandFailure from general_utils import DaosTestError diff --git a/src/tests/ftest/harness/advanced.py b/src/tests/ftest/harness/advanced.py index cc97d60e53b..f1c4a7f79be 100644 --- a/src/tests/ftest/harness/advanced.py +++ b/src/tests/ftest/harness/advanced.py @@ -6,14 +6,13 @@ import os from random import choice -from ClusterShell.NodeSet import NodeSet from apricot import TestWithServers - +from ClusterShell.NodeSet import NodeSet +from dfuse_utils import get_dfuse, start_dfuse from general_utils import get_avocado_config_value from run_utils import run_remote from test_utils_pool import POOL_TIMEOUT_INCREMENT from user_utils import get_chown_command -from dfuse_utils import get_dfuse, start_dfuse class HarnessAdvancedTest(TestWithServers): diff --git a/src/tests/ftest/harness/basic.py b/src/tests/ftest/harness/basic.py index e3997f1054a..e0e39e15c6c 100644 --- a/src/tests/ftest/harness/basic.py +++ b/src/tests/ftest/harness/basic.py @@ -9,7 +9,7 @@ from cmocka_utils import CmockaUtils from command_utils import SubProcessCommand from exception_utils import CommandFailure -from job_manager_utils import Orterun, Mpirun +from job_manager_utils import Mpirun, Orterun class HarnessBasicTest(TestWithoutServers): diff --git a/src/tests/ftest/harness/config.py b/src/tests/ftest/harness/config.py index 7904164c780..e6237c26e0b 100644 --- a/src/tests/ftest/harness/config.py +++ b/src/tests/ftest/harness/config.py @@ -4,11 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import yaml - from apricot import TestWithServers - -from general_utils import nodeset_append_suffix from dmg_utils import DmgCommand +from general_utils import nodeset_append_suffix class HarnessConfigTest(TestWithServers): diff --git a/src/tests/ftest/harness/core_files.py b/src/tests/ftest/harness/core_files.py index 4b3dac67d10..fed038a6a82 100644 --- a/src/tests/ftest/harness/core_files.py +++ b/src/tests/ftest/harness/core_files.py @@ -7,10 +7,9 @@ from random import choice from re import findall -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers -from run_utils import run_remote, run_local, RunException +from ClusterShell.NodeSet import NodeSet +from run_utils import RunException, run_local, run_remote class HarnessCoreFilesTest(TestWithServers): diff --git a/src/tests/ftest/harness/launch_setup.py b/src/tests/ftest/harness/launch_setup.py index 758cafcf891..e4dc1ae0282 100644 --- a/src/tests/ftest/harness/launch_setup.py +++ b/src/tests/ftest/harness/launch_setup.py @@ -4,8 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from apricot import TestWithoutServers - -from run_utils import run_remote, command_as_user +from run_utils import command_as_user, run_remote from user_utils import get_group_id, get_user_groups diff --git a/src/tests/ftest/harness/skip_list.py b/src/tests/ftest/harness/skip_list.py index e8a73bc959f..fc3e7023cf9 100644 --- a/src/tests/ftest/harness/skip_list.py +++ b/src/tests/ftest/harness/skip_list.py @@ -3,8 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import os import errno +import os + from apricot import Test diff --git a/src/tests/ftest/harness/slurm.py b/src/tests/ftest/harness/slurm.py index a552a662ed9..7924f8b5574 100644 --- a/src/tests/ftest/harness/slurm.py +++ b/src/tests/ftest/harness/slurm.py @@ -4,7 +4,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from apricot import TestWithoutServers - from host_utils import get_local_host from slurm_utils import sinfo diff --git a/src/tests/ftest/harness/unit.py b/src/tests/ftest/harness/unit.py index 51828a0d03c..1f4524674f6 100644 --- a/src/tests/ftest/harness/unit.py +++ b/src/tests/ftest/harness/unit.py @@ -3,12 +3,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithoutServers -from data_utils import list_unique, list_flatten, list_stats, \ - dict_extract_values, dict_subtract -from run_utils import run_remote, ResultData +from ClusterShell.NodeSet import NodeSet +from data_utils import dict_extract_values, dict_subtract, list_flatten, list_stats, list_unique +from run_utils import ResultData, run_remote class HarnessUnitTest(TestWithoutServers): diff --git a/src/tests/ftest/interoperability/upgrade_downgrade_base.py b/src/tests/ftest/interoperability/upgrade_downgrade_base.py index a168ab1a4d8..bc295372115 100644 --- a/src/tests/ftest/interoperability/upgrade_downgrade_base.py +++ b/src/tests/ftest/interoperability/upgrade_downgrade_base.py @@ -3,17 +3,17 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' +import base64 import os -import traceback import random -import base64 import time +import traceback -from pydaos.raw import DaosApiError -from general_utils import get_random_bytes, pcmd, run_pcmd from agent_utils import include_local_host from command_utils_base import CommandFailure +from general_utils import get_random_bytes, pcmd, run_pcmd from ior_test_base import IorTestBase +from pydaos.raw import DaosApiError class UpgradeDowngradeBase(IorTestBase): diff --git a/src/tests/ftest/io/macsio_test.py b/src/tests/ftest/io/macsio_test.py index 95b003378b9..01daf179712 100644 --- a/src/tests/ftest/io/macsio_test.py +++ b/src/tests/ftest/io/macsio_test.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from general_utils import list_to_str from dfuse_test_base import DfuseTestBase +from general_utils import list_to_str from macsio_test_base import MacsioTestBase diff --git a/src/tests/ftest/io/parallel_io.py b/src/tests/ftest/io/parallel_io.py index db6bfa0df65..d4dbcae06a7 100644 --- a/src/tests/ftest/io/parallel_io.py +++ b/src/tests/ftest/io/parallel_io.py @@ -4,12 +4,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import threading +import os import subprocess # nosec +import threading import time from getpass import getuser -import os - from exception_utils import CommandFailure from fio_test_base import FioBase diff --git a/src/tests/ftest/ior/crash.py b/src/tests/ftest/ior/crash.py index a999e97d5d8..a41dc920f62 100644 --- a/src/tests/ftest/ior/crash.py +++ b/src/tests/ftest/ior/crash.py @@ -6,8 +6,8 @@ import time -from ior_test_base import IorTestBase from dmg_utils import check_system_query_status +from ior_test_base import IorTestBase class IorCrash(IorTestBase): @@ -26,6 +26,7 @@ def cont_nhandles_match(self, exp_nhandles=1, attempts=5, delay_sec=2): for _ in range(attempts): if self.container.check_container_info(ci_nhandles=exp_nhandles): return True + self.log.info("check_container_info does not match yet, sleep %d sec", delay_sec) time.sleep(delay_sec) return False @@ -97,5 +98,7 @@ def test_ior_crash(self): self.fail("One or more engines crashed") # Verify container handle opened by ior is closed (by ior before its graceful exit) - self.assertTrue(self.cont_nhandles_match(attempts=1, delay_sec=0), + # Give ior some time to get started and open the container! + # And, expect 2 open handles, one for this container open/query, and another for ior itself + self.assertTrue(self.cont_nhandles_match(exp_nhandles=2, attempts=5, delay_sec=2), "Error confirming container info nhandles") diff --git a/src/tests/ftest/ior/intercept_messages.py b/src/tests/ftest/ior/intercept_messages.py index 146e623aa17..a431f2fe47b 100644 --- a/src/tests/ftest/ior/intercept_messages.py +++ b/src/tests/ftest/ior/intercept_messages.py @@ -4,8 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import re import os +import re + from ior_test_base import IorTestBase diff --git a/src/tests/ftest/ior/intercept_messages_pil4dfs.py b/src/tests/ftest/ior/intercept_messages_pil4dfs.py index 3cbecf21d09..854907c3f8a 100644 --- a/src/tests/ftest/ior/intercept_messages_pil4dfs.py +++ b/src/tests/ftest/ior/intercept_messages_pil4dfs.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import re import os +import re from ior_test_base import IorTestBase diff --git a/src/tests/ftest/ior/small.py b/src/tests/ftest/ior/small.py index e5efab32da1..e7cb58e2dfd 100644 --- a/src/tests/ftest/ior/small.py +++ b/src/tests/ftest/ior/small.py @@ -4,7 +4,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from avocado.core.exceptions import TestFail - from ior_test_base import IorTestBase diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index dc5f9f84247..5650622bf20 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -4,6 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +# pylint: disable=too-many-lines + import getpass import json import logging diff --git a/src/tests/ftest/network/cart_self_test.py b/src/tests/ftest/network/cart_self_test.py index 003fa17ac99..c3421ed9539 100644 --- a/src/tests/ftest/network/cart_self_test.py +++ b/src/tests/ftest/network/cart_self_test.py @@ -4,10 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from apricot import TestWithServers - +from command_utils import ExecutableCommand from command_utils_base import EnvironmentVariables, FormattedParameter from exception_utils import CommandFailure -from command_utils import ExecutableCommand from host_utils import get_local_host from job_manager_utils import get_job_manager diff --git a/src/tests/ftest/nvme/fault.py b/src/tests/ftest/nvme/fault.py index 984bcaccf2f..5cb30c3815e 100644 --- a/src/tests/ftest/nvme/fault.py +++ b/src/tests/ftest/nvme/fault.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from nvme_utils import ServerFillUp from exception_utils import CommandFailure +from nvme_utils import ServerFillUp class NvmeFault(ServerFillUp): diff --git a/src/tests/ftest/nvme/fragmentation.py b/src/tests/ftest/nvme/fragmentation.py index 5bda262538c..65dd3406ef6 100644 --- a/src/tests/ftest/nvme/fragmentation.py +++ b/src/tests/ftest/nvme/fragmentation.py @@ -3,16 +3,16 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import queue import time -from threading import Thread from itertools import product -import queue +from threading import Thread from apricot import TestWithServers -from write_host_file import write_host_file -from ior_utils import IorCommand from exception_utils import CommandFailure +from ior_utils import IorCommand from job_manager_utils import get_job_manager +from write_host_file import write_host_file class NvmeFragmentation(TestWithServers): diff --git a/src/tests/ftest/nvme/health.py b/src/tests/ftest/nvme/health.py index cf06704ea0c..f1de78a7546 100644 --- a/src/tests/ftest/nvme/health.py +++ b/src/tests/ftest/nvme/health.py @@ -7,10 +7,9 @@ from avocado import fail_on from avocado.core.exceptions import TestFail - -from dmg_utils import get_storage_query_pool_info, get_dmg_smd_info -from nvme_utils import ServerFillUp, get_device_ids +from dmg_utils import get_dmg_smd_info, get_storage_query_pool_info from exception_utils import CommandFailure +from nvme_utils import ServerFillUp, get_device_ids class NvmeHealth(ServerFillUp): diff --git a/src/tests/ftest/nvme/io_verification.py b/src/tests/ftest/nvme/io_verification.py index c513ac6cefb..bd660fd2f8a 100644 --- a/src/tests/ftest/nvme/io_verification.py +++ b/src/tests/ftest/nvme/io_verification.py @@ -4,10 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import avocado - -from pydaos.raw import DaosApiError -from ior_test_base import IorTestBase from dmg_utils import check_system_query_status +from ior_test_base import IorTestBase +from pydaos.raw import DaosApiError class NvmeIoVerification(IorTestBase): diff --git a/src/tests/ftest/nvme/object.py b/src/tests/ftest/nvme/object.py index 516c17c0ffd..4c473401731 100644 --- a/src/tests/ftest/nvme/object.py +++ b/src/tests/ftest/nvme/object.py @@ -4,12 +4,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import avocado - -from pydaos.raw import DaosApiError - from apricot import TestWithServers -from thread_manager import ThreadManager from general_utils import report_errors +from pydaos.raw import DaosApiError +from thread_manager import ThreadManager class NvmeObject(TestWithServers): diff --git a/src/tests/ftest/nvme/pool_capacity.py b/src/tests/ftest/nvme/pool_capacity.py index 815ab1910f0..2f88d012092 100644 --- a/src/tests/ftest/nvme/pool_capacity.py +++ b/src/tests/ftest/nvme/pool_capacity.py @@ -3,16 +3,16 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time +import queue import threading +import time from itertools import product -import queue from apricot import TestWithServers -from write_host_file import write_host_file +from exception_utils import CommandFailure from ior_utils import IorCommand from job_manager_utils import get_job_manager -from exception_utils import CommandFailure +from write_host_file import write_host_file class NvmePoolCapacity(TestWithServers): diff --git a/src/tests/ftest/nvme/pool_exclude.py b/src/tests/ftest/nvme/pool_exclude.py index aad66411fb3..578070c1c47 100644 --- a/src/tests/ftest/nvme/pool_exclude.py +++ b/src/tests/ftest/nvme/pool_exclude.py @@ -3,17 +3,17 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from multiprocessing import Queue -import time import random -import threading import re +import threading +import time +from multiprocessing import Queue from exception_utils import CommandFailure from ior_utils import run_ior, thread_run_ior from job_manager_utils import get_job_manager -from test_utils_pool import add_pool from osa_utils import OSAUtils +from test_utils_pool import add_pool from write_host_file import write_host_file diff --git a/src/tests/ftest/nvme/pool_extend.py b/src/tests/ftest/nvme/pool_extend.py index 44ef7575ddf..18e48ced8a1 100644 --- a/src/tests/ftest/nvme/pool_extend.py +++ b/src/tests/ftest/nvme/pool_extend.py @@ -3,12 +3,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import threading +import time +from dmg_utils import check_system_query_status from osa_utils import OSAUtils from write_host_file import write_host_file -from dmg_utils import check_system_query_status class NvmePoolExtend(OSAUtils): diff --git a/src/tests/ftest/object/array.py b/src/tests/ftest/object/array.py index be7f46807ff..b14e51306de 100644 --- a/src/tests/ftest/object/array.py +++ b/src/tests/ftest/object/array.py @@ -3,13 +3,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' +import logging import time import traceback -import logging - -from pydaos.raw import DaosContainer, DaosApiError, c_uuid_to_str from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContainer, c_uuid_to_str class ArrayObjTest(TestWithServers): diff --git a/src/tests/ftest/object/create_many_dkeys.py b/src/tests/ftest/object/create_many_dkeys.py index 52d073fe2dd..c4feb0bdf48 100644 --- a/src/tests/ftest/object/create_many_dkeys.py +++ b/src/tests/ftest/object/create_many_dkeys.py @@ -3,14 +3,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import sys import ctypes -import avocado - -from pydaos.raw import DaosContainer, IORequest, DaosApiError +import sys +import avocado from apricot import TestWithServers from general_utils import create_string_buffer +from pydaos.raw import DaosApiError, DaosContainer, IORequest class CreateManyDkeys(TestWithServers): diff --git a/src/tests/ftest/object/fetch_bad_param.py b/src/tests/ftest/object/fetch_bad_param.py index 919fc42b831..8c2f91055f0 100644 --- a/src/tests/ftest/object/fetch_bad_param.py +++ b/src/tests/ftest/object/fetch_bad_param.py @@ -6,9 +6,8 @@ import time import traceback -from pydaos.raw import DaosContainer, DaosApiError - from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContainer class ObjFetchBadParam(TestWithServers): diff --git a/src/tests/ftest/object/integrity.py b/src/tests/ftest/object/integrity.py index 097d7f17460..8f37422b106 100644 --- a/src/tests/ftest/object/integrity.py +++ b/src/tests/ftest/object/integrity.py @@ -5,11 +5,11 @@ """ import ctypes import time -import avocado -from pydaos.raw import (DaosContainer, IORequest, DaosObj, DaosApiError) +import avocado from apricot import TestWithServers from general_utils import create_string_buffer +from pydaos.raw import DaosApiError, DaosContainer, DaosObj, IORequest class ObjectDataValidation(TestWithServers): diff --git a/src/tests/ftest/object/open_bad_param.py b/src/tests/ftest/object/open_bad_param.py index 045fcdd0332..8815d6e4aa2 100644 --- a/src/tests/ftest/object/open_bad_param.py +++ b/src/tests/ftest/object/open_bad_param.py @@ -5,9 +5,8 @@ """ import traceback -from pydaos.raw import DaosContainer, DaosApiError, DaosObjId - from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContainer, DaosObjId class ObjOpenBadParam(TestWithServers): diff --git a/src/tests/ftest/object/punch_test.py b/src/tests/ftest/object/punch_test.py index a55fd8dad03..ca1d300a07d 100644 --- a/src/tests/ftest/object/punch_test.py +++ b/src/tests/ftest/object/punch_test.py @@ -5,9 +5,8 @@ ''' import traceback -from pydaos.raw import DaosContainer, DaosApiError - from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContainer class PunchTest(TestWithServers): diff --git a/src/tests/ftest/object/same_key_different_value.py b/src/tests/ftest/object/same_key_different_value.py index e37c69456d1..a35222c21f4 100644 --- a/src/tests/ftest/object/same_key_different_value.py +++ b/src/tests/ftest/object/same_key_different_value.py @@ -3,9 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from pydaos.raw import DaosApiError - from apricot import TestWithServers +from pydaos.raw import DaosApiError class SameKeyDifferentValue(TestWithServers): diff --git a/src/tests/ftest/object/update_bad_param.py b/src/tests/ftest/object/update_bad_param.py index 6462bdd09ba..5a58a0292af 100644 --- a/src/tests/ftest/object/update_bad_param.py +++ b/src/tests/ftest/object/update_bad_param.py @@ -5,9 +5,8 @@ ''' import traceback -from pydaos.raw import DaosContainer, DaosApiError - from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContainer class ObjUpdateBadParam(TestWithServers): diff --git a/src/tests/ftest/osa/offline_drain.py b/src/tests/ftest/osa/offline_drain.py index 0a67edc2f7f..26be76d95a8 100644 --- a/src/tests/ftest/osa/offline_drain.py +++ b/src/tests/ftest/osa/offline_drain.py @@ -5,9 +5,9 @@ """ import random +from nvme_utils import ServerFillUp from osa_utils import OSAUtils from test_utils_pool import add_pool -from nvme_utils import ServerFillUp from write_host_file import write_host_file diff --git a/src/tests/ftest/osa/offline_extend.py b/src/tests/ftest/osa/offline_extend.py index 0f8bcc35fe5..471bdc64fd5 100644 --- a/src/tests/ftest/osa/offline_extend.py +++ b/src/tests/ftest/osa/offline_extend.py @@ -3,9 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from dmg_utils import check_system_query_status from osa_utils import OSAUtils from test_utils_pool import add_pool -from dmg_utils import check_system_query_status class OSAOfflineExtend(OSAUtils): diff --git a/src/tests/ftest/osa/offline_parallel_test.py b/src/tests/ftest/osa/offline_parallel_test.py index 8dd19354dbb..c0d2c7f14a4 100644 --- a/src/tests/ftest/osa/offline_parallel_test.py +++ b/src/tests/ftest/osa/offline_parallel_test.py @@ -3,13 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time -import random -import threading import copy import queue -from osa_utils import OSAUtils +import random +import threading +import time + from dmg_utils import check_system_query_status +from osa_utils import OSAUtils from test_utils_pool import add_pool diff --git a/src/tests/ftest/osa/offline_reintegration.py b/src/tests/ftest/osa/offline_reintegration.py index dab5f237d8b..6491a477d7a 100644 --- a/src/tests/ftest/osa/offline_reintegration.py +++ b/src/tests/ftest/osa/offline_reintegration.py @@ -5,8 +5,8 @@ """ import random -from osa_utils import OSAUtils from nvme_utils import ServerFillUp +from osa_utils import OSAUtils from test_utils_pool import add_pool from write_host_file import write_host_file diff --git a/src/tests/ftest/osa/online_drain.py b/src/tests/ftest/osa/online_drain.py index bd4c26b0b53..bf5804c3767 100644 --- a/src/tests/ftest/osa/online_drain.py +++ b/src/tests/ftest/osa/online_drain.py @@ -3,13 +3,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import random import threading +import time -from write_host_file import write_host_file from osa_utils import OSAUtils from test_utils_pool import add_pool +from write_host_file import write_host_file class OSAOnlineDrain(OSAUtils): diff --git a/src/tests/ftest/osa/online_extend.py b/src/tests/ftest/osa/online_extend.py index a2d6569c528..bb17ecbd388 100644 --- a/src/tests/ftest/osa/online_extend.py +++ b/src/tests/ftest/osa/online_extend.py @@ -3,14 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import threading +import time -from test_utils_pool import add_pool -from write_host_file import write_host_file from daos_racer_utils import DaosRacerCommand from dmg_utils import check_system_query_status from osa_utils import OSAUtils +from test_utils_pool import add_pool +from write_host_file import write_host_file class OSAOnlineExtend(OSAUtils): diff --git a/src/tests/ftest/osa/online_parallel_test.py b/src/tests/ftest/osa/online_parallel_test.py index 6d8f36e35b7..c99a45dbd66 100644 --- a/src/tests/ftest/osa/online_parallel_test.py +++ b/src/tests/ftest/osa/online_parallel_test.py @@ -3,18 +3,18 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time +import copy +import queue import random import threading -import copy - +import time from itertools import product -import queue -from write_host_file import write_host_file -from exception_utils import CommandFailure + +from apricot import skipForTicket from daos_racer_utils import DaosRacerCommand +from exception_utils import CommandFailure from osa_utils import OSAUtils -from apricot import skipForTicket +from write_host_file import write_host_file class OSAOnlineParallelTest(OSAUtils): diff --git a/src/tests/ftest/osa/online_reintegration.py b/src/tests/ftest/osa/online_reintegration.py index 106e9bda2a9..3e4cda5eb3c 100644 --- a/src/tests/ftest/osa/online_reintegration.py +++ b/src/tests/ftest/osa/online_reintegration.py @@ -3,15 +3,15 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time +import queue import random import threading -import queue +import time -from test_utils_pool import add_pool -from write_host_file import write_host_file from daos_racer_utils import DaosRacerCommand from osa_utils import OSAUtils +from test_utils_pool import add_pool +from write_host_file import write_host_file class OSAOnlineReintegration(OSAUtils): diff --git a/src/tests/ftest/performance/mdtest_hard.py b/src/tests/ftest/performance/mdtest_hard.py index 971a2aad471..1d6107670ad 100644 --- a/src/tests/ftest/performance/mdtest_hard.py +++ b/src/tests/ftest/performance/mdtest_hard.py @@ -5,6 +5,7 @@ ''' import os + from performance_test_base import PerformanceTestBase diff --git a/src/tests/ftest/pool/api_attribute.py b/src/tests/ftest/pool/api_attribute.py index d2e0541ba56..f83f6e1e370 100644 --- a/src/tests/ftest/pool/api_attribute.py +++ b/src/tests/ftest/pool/api_attribute.py @@ -5,10 +5,9 @@ ''' import traceback -from pydaos.raw import DaosApiError - from apricot import TestWithServers from general_utils import get_random_bytes +from pydaos.raw import DaosApiError from test_utils_base import CallbackHandler diff --git a/src/tests/ftest/pool/bad_connect.py b/src/tests/ftest/pool/bad_connect.py index 69ac2974651..4d524259c0c 100644 --- a/src/tests/ftest/pool/bad_connect.py +++ b/src/tests/ftest/pool/bad_connect.py @@ -3,10 +3,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import traceback import ctypes -from avocado.core.exceptions import TestFail +import traceback + from apricot import TestWithServers +from avocado.core.exceptions import TestFail class BadConnectTest(TestWithServers): diff --git a/src/tests/ftest/pool/bad_query.py b/src/tests/ftest/pool/bad_query.py index 0b212aaa5b4..964cba97209 100644 --- a/src/tests/ftest/pool/bad_query.py +++ b/src/tests/ftest/pool/bad_query.py @@ -5,9 +5,8 @@ ''' import traceback -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail class BadQueryTest(TestWithServers): diff --git a/src/tests/ftest/pool/create.py b/src/tests/ftest/pool/create.py index 787363ddc3e..433dd88110d 100644 --- a/src/tests/ftest/pool/create.py +++ b/src/tests/ftest/pool/create.py @@ -4,7 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from apricot import TestWithServers -from test_utils_pool import add_pool, get_size_params, check_pool_creation +from test_utils_pool import add_pool, check_pool_creation, get_size_params class PoolCreateTests(TestWithServers): diff --git a/src/tests/ftest/pool/destroy.py b/src/tests/ftest/pool/destroy.py index a7d5b192925..cec6ef7c9f9 100644 --- a/src/tests/ftest/pool/destroy.py +++ b/src/tests/ftest/pool/destroy.py @@ -5,11 +5,10 @@ """ import os -from avocado.core.exceptions import TestFail - from apricot import TestWithServers -from general_utils import get_default_config_file, check_pool_files +from avocado.core.exceptions import TestFail from dmg_utils import get_dmg_command +from general_utils import check_pool_files, get_default_config_file class DestroyTests(TestWithServers): diff --git a/src/tests/ftest/pool/dynamic_server_pool.py b/src/tests/ftest/pool/dynamic_server_pool.py index 2d48f25e3ad..c4f8c56b2ca 100644 --- a/src/tests/ftest/pool/dynamic_server_pool.py +++ b/src/tests/ftest/pool/dynamic_server_pool.py @@ -3,9 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers +from ClusterShell.NodeSet import NodeSet from general_utils import check_for_pool diff --git a/src/tests/ftest/pool/evict.py b/src/tests/ftest/pool/evict.py index 285fe81bd56..539d0e2a412 100644 --- a/src/tests/ftest/pool/evict.py +++ b/src/tests/ftest/pool/evict.py @@ -3,12 +3,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from apricot import TestWithServers from avocado.core.exceptions import TestFail from ClusterShell.NodeSet import NodeSet from pydaos.raw import DaosApiError, c_uuid_to_str -from apricot import TestWithServers - class PoolEvictTest(TestWithServers): """ diff --git a/src/tests/ftest/pool/label.py b/src/tests/ftest/pool/label.py index 9b6a3131846..09743aa960e 100644 --- a/src/tests/ftest/pool/label.py +++ b/src/tests/ftest/pool/label.py @@ -5,11 +5,10 @@ """ import string -from avocado.core.exceptions import TestFail - from apricot import TestWithServers -from general_utils import report_errors, get_random_string +from avocado.core.exceptions import TestFail from exception_utils import CommandFailure +from general_utils import get_random_string, report_errors class Label(TestWithServers): diff --git a/src/tests/ftest/pool/list_pools.py b/src/tests/ftest/pool/list_pools.py index c0127b155c8..b1a85a0125d 100644 --- a/src/tests/ftest/pool/list_pools.py +++ b/src/tests/ftest/pool/list_pools.py @@ -3,9 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail from exception_utils import CommandFailure diff --git a/src/tests/ftest/pool/list_verbose.py b/src/tests/ftest/pool/list_verbose.py index f60e01abd48..656ec80c5a6 100644 --- a/src/tests/ftest/pool/list_verbose.py +++ b/src/tests/ftest/pool/list_verbose.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ior_test_base import IorTestBase from general_utils import report_errors +from ior_test_base import IorTestBase class ListVerboseTest(IorTestBase): diff --git a/src/tests/ftest/pool/management_race.py b/src/tests/ftest/pool/management_race.py index 725ced287b0..1a86c31b4e5 100644 --- a/src/tests/ftest/pool/management_race.py +++ b/src/tests/ftest/pool/management_race.py @@ -3,14 +3,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import random - -from pydaos.raw import DaosApiError +import time from apricot import TestWithServers -from thread_manager import ThreadManager from command_utils_base import CommandFailure +from pydaos.raw import DaosApiError +from thread_manager import ThreadManager class PoolManagementRace(TestWithServers): diff --git a/src/tests/ftest/pool/multi_server_create_delete.py b/src/tests/ftest/pool/multi_server_create_delete.py index d2207db693f..0e2fb334cd5 100644 --- a/src/tests/ftest/pool/multi_server_create_delete.py +++ b/src/tests/ftest/pool/multi_server_create_delete.py @@ -5,12 +5,10 @@ """ import os -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers +from ClusterShell.NodeSet import NodeSet from general_utils import check_for_pool - RESULT_PASS = "PASS" # nosec RESULT_FAIL = "FAIL" diff --git a/src/tests/ftest/pool/permission.py b/src/tests/ftest/pool/permission.py index 3784a02a778..f2f247d9bba 100644 --- a/src/tests/ftest/pool/permission.py +++ b/src/tests/ftest/pool/permission.py @@ -3,10 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from pydaos.raw import DaosContainer, DaosApiError -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail +from pydaos.raw import DaosApiError, DaosContainer RESULT_PASS = "PASS" # nosec RESULT_FAIL = "FAIL" diff --git a/src/tests/ftest/pool/query_attribute.py b/src/tests/ftest/pool/query_attribute.py index e90b631f92e..988e1d18d1b 100644 --- a/src/tests/ftest/pool/query_attribute.py +++ b/src/tests/ftest/pool/query_attribute.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import base64 + from apricot import TestWithServers diff --git a/src/tests/ftest/pool/svc.py b/src/tests/ftest/pool/svc.py index f60a8ed7a7b..c459eba4e8a 100644 --- a/src/tests/ftest/pool/svc.py +++ b/src/tests/ftest/pool/svc.py @@ -3,9 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail class PoolSvc(TestWithServers): diff --git a/src/tests/ftest/pool/verify_space.py b/src/tests/ftest/pool/verify_space.py index a463c6ef5a0..1fd8469fddb 100644 --- a/src/tests/ftest/pool/verify_space.py +++ b/src/tests/ftest/pool/verify_space.py @@ -7,7 +7,6 @@ import re from apricot import TestWithServers - from exception_utils import CommandFailure from general_utils import human_to_bytes from ior_utils import run_ior diff --git a/src/tests/ftest/rebuild/cascading_failures.py b/src/tests/ftest/rebuild/cascading_failures.py index 4edd0064b6a..a17c6c6ffac 100644 --- a/src/tests/ftest/rebuild/cascading_failures.py +++ b/src/tests/ftest/rebuild/cascading_failures.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from rebuild_test_base import RebuildTestBase from daos_utils import DaosCommand +from rebuild_test_base import RebuildTestBase class RbldCascadingFailures(RebuildTestBase): diff --git a/src/tests/ftest/rebuild/container_create_race.py b/src/tests/ftest/rebuild/container_create_race.py index 6684d89ad53..c98103dd00d 100644 --- a/src/tests/ftest/rebuild/container_create_race.py +++ b/src/tests/ftest/rebuild/container_create_race.py @@ -4,7 +4,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from avocado.core.exceptions import TestFail - from ior_test_base import IorTestBase diff --git a/src/tests/ftest/rebuild/no_cap.py b/src/tests/ftest/rebuild/no_cap.py index 8bb773bb1a5..5b7f81fed00 100644 --- a/src/tests/ftest/rebuild/no_cap.py +++ b/src/tests/ftest/rebuild/no_cap.py @@ -5,8 +5,9 @@ ''' from time import sleep, time + from apricot import TestWithServers -from general_utils import get_random_bytes, DaosTestError +from general_utils import DaosTestError, get_random_bytes from test_utils_container import TestContainerData diff --git a/src/tests/ftest/rebuild/no_cap.yaml b/src/tests/ftest/rebuild/no_cap.yaml index 25aca5676b1..f55793eebf9 100644 --- a/src/tests/ftest/rebuild/no_cap.yaml +++ b/src/tests/ftest/rebuild/no_cap.yaml @@ -12,7 +12,7 @@ server_config: engines_per_host: 2 engines: 0: - targets: 8 + targets: 1 pinned_numa_node: 0 nr_xs_helpers: 1 fabric_iface: ib0 @@ -20,7 +20,7 @@ server_config: log_file: daos_server0.log storage: auto 1: - targets: 8 + targets: 1 pinned_numa_node: 1 nr_xs_helpers: 1 fabric_iface: ib1 @@ -36,6 +36,6 @@ pool: pool_query_timeout: 30 pool_query_interval: 1 test_data_list: [1048576] - oclass: "OC_RP_2G1" + oclass: "OC_RP_4G1" rebuild: rank_to_kill: 1 diff --git a/src/tests/ftest/rebuild/read_array.py b/src/tests/ftest/rebuild/read_array.py index 58207839a17..6b72bf45bbd 100644 --- a/src/tests/ftest/rebuild/read_array.py +++ b/src/tests/ftest/rebuild/read_array.py @@ -3,9 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from rebuild_test_base import RebuildTestBase from daos_utils import DaosCommand from general_utils import DaosTestError +from rebuild_test_base import RebuildTestBase class RbldReadArrayTest(RebuildTestBase): diff --git a/src/tests/ftest/rebuild/widely_striped.py b/src/tests/ftest/rebuild/widely_striped.py index 01eee76014c..470926df05d 100644 --- a/src/tests/ftest/rebuild/widely_striped.py +++ b/src/tests/ftest/rebuild/widely_striped.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import time + from mdtest_test_base import MdtestBase diff --git a/src/tests/ftest/scrubber/csum_fault.py b/src/tests/ftest/scrubber/csum_fault.py index 6e1b9a968db..4003f6a13f1 100644 --- a/src/tests/ftest/scrubber/csum_fault.py +++ b/src/tests/ftest/scrubber/csum_fault.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import time + from scrubber_test_base import TestWithScrubber diff --git a/src/tests/ftest/security/cont_acl.py b/src/tests/ftest/security/cont_acl.py index dd3a6a69733..166da1e459b 100644 --- a/src/tests/ftest/security/cont_acl.py +++ b/src/tests/ftest/security/cont_acl.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os + import security_test_base as secTestBase from cont_security_test_base import ContSecurityTestBase from pool_security_test_base import PoolSecurityTestBase diff --git a/src/tests/ftest/security/cont_delete_acl.py b/src/tests/ftest/security/cont_delete_acl.py index 3415c7e2b6a..a8b729fc357 100644 --- a/src/tests/ftest/security/cont_delete_acl.py +++ b/src/tests/ftest/security/cont_delete_acl.py @@ -4,7 +4,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from avocado import fail_on - from cont_security_test_base import ContSecurityTestBase from exception_utils import CommandFailure diff --git a/src/tests/ftest/security/cont_get_acl.py b/src/tests/ftest/security/cont_get_acl.py index 86a675fab5d..b703d7e8d93 100644 --- a/src/tests/ftest/security/cont_get_acl.py +++ b/src/tests/ftest/security/cont_get_acl.py @@ -6,10 +6,9 @@ import os from avocado import fail_on - from cont_security_test_base import ContSecurityTestBase -from security_test_base import read_acl_file from exception_utils import CommandFailure +from security_test_base import read_acl_file class GetContainerACLTest(ContSecurityTestBase): diff --git a/src/tests/ftest/security/cont_overwrite_acl.py b/src/tests/ftest/security/cont_overwrite_acl.py index 9a494c58ebf..10de9a06883 100644 --- a/src/tests/ftest/security/cont_overwrite_acl.py +++ b/src/tests/ftest/security/cont_overwrite_acl.py @@ -6,10 +6,9 @@ import os from avocado import fail_on - from cont_security_test_base import ContSecurityTestBase -from security_test_base import create_acl_file from exception_utils import CommandFailure +from security_test_base import create_acl_file class OverwriteContainerACLTest(ContSecurityTestBase): diff --git a/src/tests/ftest/security/cont_update_acl.py b/src/tests/ftest/security/cont_update_acl.py index c0f971dd260..82a4922ff97 100644 --- a/src/tests/ftest/security/cont_update_acl.py +++ b/src/tests/ftest/security/cont_update_acl.py @@ -6,10 +6,9 @@ import os from avocado import fail_on - from cont_security_test_base import ContSecurityTestBase -from security_test_base import create_acl_file from exception_utils import CommandFailure +from security_test_base import create_acl_file class UpdateContainerACLTest(ContSecurityTestBase): diff --git a/src/tests/ftest/security/pool_acl.py b/src/tests/ftest/security/pool_acl.py index d46635083b4..5d8cc6728c2 100644 --- a/src/tests/ftest/security/pool_acl.py +++ b/src/tests/ftest/security/pool_acl.py @@ -3,12 +3,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' +import grp import os import pwd -import grp -from security_test_base import acl_entry from pool_security_test_base import PoolSecurityTestBase +from security_test_base import acl_entry PERMISSIONS = ["", "r", "w", "rw"] diff --git a/src/tests/ftest/security/pool_connect_init.py b/src/tests/ftest/security/pool_connect_init.py index 70ffd5b33fc..4ddd30ddacb 100644 --- a/src/tests/ftest/security/pool_connect_init.py +++ b/src/tests/ftest/security/pool_connect_init.py @@ -6,9 +6,8 @@ import os import traceback -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail class PoolSecurityTest(TestWithServers): diff --git a/src/tests/ftest/security/pool_groups.py b/src/tests/ftest/security/pool_groups.py index fe02295b75d..f16b9863796 100644 --- a/src/tests/ftest/security/pool_groups.py +++ b/src/tests/ftest/security/pool_groups.py @@ -3,11 +3,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import os import grp +import os -from security_test_base import acl_entry from pool_security_test_base import PoolSecurityTestBase +from security_test_base import acl_entry PERMISSIONS = ["", "r", "w", "rw"] diff --git a/src/tests/ftest/server/daos_server_restart.py b/src/tests/ftest/server/daos_server_restart.py index ce58fcf29f0..be3583f9459 100644 --- a/src/tests/ftest/server/daos_server_restart.py +++ b/src/tests/ftest/server/daos_server_restart.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from avocado import fail_on from apricot import TestWithServers +from avocado import fail_on from exception_utils import CommandFailure from server_utils import ServerFailed diff --git a/src/tests/ftest/server/metadata.py b/src/tests/ftest/server/metadata.py index 50ef7991a99..18cf9edf70f 100644 --- a/src/tests/ftest/server/metadata.py +++ b/src/tests/ftest/server/metadata.py @@ -3,14 +3,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import traceback import time - -from avocado.core.exceptions import TestFail +import traceback from apricot import TestWithServers -from ior_utils import IorCommand +from avocado.core.exceptions import TestFail from exception_utils import CommandFailure +from ior_utils import IorCommand from job_manager_utils import get_job_manager from thread_manager import ThreadManager diff --git a/src/tests/ftest/server/multiengine_persocket.py b/src/tests/ftest/server/multiengine_persocket.py index f838610e91c..7c897415fe9 100644 --- a/src/tests/ftest/server/multiengine_persocket.py +++ b/src/tests/ftest/server/multiengine_persocket.py @@ -6,14 +6,13 @@ import base64 import traceback -from pydaos.raw import DaosApiError - -from general_utils import get_random_bytes, wait_for_result, check_ping, check_ssh -from run_utils import run_remote, run_local +from general_utils import check_ping, check_ssh, get_random_bytes, wait_for_result from ior_test_base import IorTestBase from mdtest_test_base import MdtestBase +from pydaos.raw import DaosApiError +from run_utils import run_local, run_remote from server_utils_base import DaosServerCommand -from storage_utils import StorageInfo, StorageException +from storage_utils import StorageException, StorageInfo class MultiEnginesPerSocketTest(IorTestBase, MdtestBase): diff --git a/src/tests/ftest/server/replay.py b/src/tests/ftest/server/replay.py index ce7cf569952..1b9f08b114c 100644 --- a/src/tests/ftest/server/replay.py +++ b/src/tests/ftest/server/replay.py @@ -7,7 +7,6 @@ import time from apricot import TestWithServers - from dfuse_utils import get_dfuse, start_dfuse, stop_dfuse from general_utils import join from ior_utils import get_ior diff --git a/src/tests/ftest/server/storage_tiers.py b/src/tests/ftest/server/storage_tiers.py index 7db0e495e5b..e627b8d62c2 100644 --- a/src/tests/ftest/server/storage_tiers.py +++ b/src/tests/ftest/server/storage_tiers.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os -import yaml +import yaml from apricot import TestWithServers from command_utils_base import CommonConfig from server_utils import DaosServerTransportCredentials, DaosServerYamlParameters diff --git a/src/tests/ftest/telemetry/dkey_akey_enum_punch.py b/src/tests/ftest/telemetry/dkey_akey_enum_punch.py index b88d1b0c5c2..99481406d8e 100644 --- a/src/tests/ftest/telemetry/dkey_akey_enum_punch.py +++ b/src/tests/ftest/telemetry/dkey_akey_enum_punch.py @@ -5,10 +5,9 @@ ''' import ctypes -from pydaos.raw import DaosContainer, IORequest, DaosObjClass - -from telemetry_test_base import TestWithTelemetry from general_utils import create_string_buffer +from pydaos.raw import DaosContainer, DaosObjClass, IORequest +from telemetry_test_base import TestWithTelemetry class DkeyAkeyEnumPunch(TestWithTelemetry): diff --git a/src/tests/ftest/util/agent_utils.py b/src/tests/ftest/util/agent_utils.py index 4252d5aeb3d..dec6f0d2e3a 100644 --- a/src/tests/ftest/util/agent_utils.py +++ b/src/tests/ftest/util/agent_utils.py @@ -3,21 +3,17 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import socket -import re import os +import re +import socket from getpass import getuser +from agent_utils_params import DaosAgentTransportCredentials, DaosAgentYamlParameters from ClusterShell.NodeSet import NodeSet - -from command_utils_base import \ - FormattedParameter, EnvironmentVariables, \ - CommonConfig +from command_utils import CommandWithSubCommand, SubprocessManager, YamlCommand +from command_utils_base import CommonConfig, EnvironmentVariables, FormattedParameter from exception_utils import CommandFailure -from command_utils import YamlCommand, CommandWithSubCommand, SubprocessManager from general_utils import get_log_file, run_pcmd -from agent_utils_params import \ - DaosAgentTransportCredentials, DaosAgentYamlParameters def include_local_host(hosts): diff --git a/src/tests/ftest/util/agent_utils_params.py b/src/tests/ftest/util/agent_utils_params.py index c16dedfd443..46b793f31ef 100644 --- a/src/tests/ftest/util/agent_utils_params.py +++ b/src/tests/ftest/util/agent_utils_params.py @@ -5,7 +5,7 @@ """ import os -from command_utils_base import BasicParameter, LogParameter, YamlParameters, TransportCredentials +from command_utils_base import BasicParameter, LogParameter, TransportCredentials, YamlParameters class DaosAgentTransportCredentials(TransportCredentials): diff --git a/src/tests/ftest/util/apricot/apricot/__init__.py b/src/tests/ftest/util/apricot/apricot/__init__.py index 1d7b29f52c9..ad62cf1297a 100644 --- a/src/tests/ftest/util/apricot/apricot/__init__.py +++ b/src/tests/ftest/util/apricot/apricot/__init__.py @@ -1,5 +1,4 @@ """apricot __init__.""" __all__ = ['Test', 'TestWithServers', 'TestWithoutServers', 'skipForTicket'] -from apricot.test import Test, TestWithServers, TestWithoutServers -from apricot.test import skipForTicket +from apricot.test import Test, TestWithoutServers, TestWithServers, skipForTicket diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index f0996196951..aed0ea77ce5 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -5,41 +5,39 @@ """ # pylint: disable=too-many-lines -from ast import literal_eval -import os import json +import os +import random import re import sys +from ast import literal_eval from time import time -import random -from avocado import fail_on, skip, TestFail +from agent_utils import DaosAgentManager, include_local_host from avocado import Test as avocadoTest +from avocado import TestFail, fail_on, skip from avocado.core import exceptions -from ClusterShell.NodeSet import NodeSet -from pydaos.raw import DaosContext, DaosLog, DaosApiError - -from agent_utils import DaosAgentManager, include_local_host from cart_ctl_utils import CartCtl +from ClusterShell.NodeSet import NodeSet from command_utils_base import EnvironmentVariables -from exception_utils import CommandFailure from daos_utils import DaosCommand from distro_utils import detect from dmg_utils import get_dmg_command +from exception_utils import CommandFailure from fault_config_utils import FaultInjection -from general_utils import \ - get_default_config_file, pcmd, get_file_listing, DaosTestError, run_command, \ - dump_engines_stacks, get_avocado_config_value, set_avocado_config_value, \ - nodeset_append_suffix, dict_to_str -from host_utils import get_local_host, get_host_parameters, HostRole, HostInfo, HostException +from general_utils import (DaosTestError, dict_to_str, dump_engines_stacks, + get_avocado_config_value, get_default_config_file, get_file_listing, + nodeset_append_suffix, pcmd, run_command, set_avocado_config_value) +from host_utils import HostException, HostInfo, HostRole, get_host_parameters, get_local_host +from job_manager_utils import get_job_manager from logger_utils import TestLogger +from pydaos.raw import DaosApiError, DaosContext, DaosLog +from run_utils import command_as_user, run_remote, stop_processes from server_utils import DaosServerManager -from run_utils import stop_processes, run_remote, command_as_user -from slurm_utils import get_partition_hosts, get_reservation_hosts, SlurmFailed +from slurm_utils import SlurmFailed, get_partition_hosts, get_reservation_hosts from test_utils_container import TestContainer -from test_utils_pool import LabelGenerator, add_pool, POOL_NAMESPACE +from test_utils_pool import POOL_NAMESPACE, LabelGenerator, add_pool from write_host_file import write_host_file -from job_manager_utils import get_job_manager def skipForTicket(ticket): # pylint: disable=invalid-name diff --git a/src/tests/ftest/util/cart_ctl_utils.py b/src/tests/ftest/util/cart_ctl_utils.py index 5b5182bf21c..4d6272bb21a 100644 --- a/src/tests/ftest/util/cart_ctl_utils.py +++ b/src/tests/ftest/util/cart_ctl_utils.py @@ -4,9 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from command_utils_base import FormattedParameter -from command_utils_base import BasicParameter from command_utils import ExecutableCommand +from command_utils_base import BasicParameter, FormattedParameter # pylint: disable=too-few-public-methods,too-many-instance-attributes diff --git a/src/tests/ftest/util/cmocka_utils.py b/src/tests/ftest/util/cmocka_utils.py index 78b8614daff..4bda40e9e95 100644 --- a/src/tests/ftest/util/cmocka_utils.py +++ b/src/tests/ftest/util/cmocka_utils.py @@ -9,7 +9,7 @@ from command_utils import ExecutableCommand from command_utils_base import EnvironmentVariables from exception_utils import CommandFailure -from results_utils import TestName, TestResult, Results, Job, create_xml +from results_utils import Job, Results, TestName, TestResult, create_xml from run_utils import get_clush_command, run_local, run_remote diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index b8e996cfc25..f160af869c9 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -9,6 +9,7 @@ import re import sys from collections import OrderedDict +from difflib import unified_diff from ClusterShell.NodeSet import NodeSet from process_core_files import CoreFileException, CoreFileProcessing @@ -870,12 +871,6 @@ def replace_xml(logger, xml_file, pattern, replacement, xml_data, test_result): str: the updated xml_data; None if an error was detected """ logger.debug("Replacing '%s' with '%s' in %s", pattern, replacement, xml_file) - - logger.debug(" Contents of %s before replacement", xml_file) - for line in xml_data.splitlines(): - logger.debug(" %s", line) - logger.debug("") - try: with open(xml_file, "w", encoding="utf-8") as xml_buffer: xml_buffer.write(re.sub(pattern, replacement, xml_data)) @@ -886,8 +881,9 @@ def replace_xml(logger, xml_file, pattern, replacement, xml_data, test_result): new_xml_data = get_xml_data(logger, xml_file, test_result) if new_xml_data is not None: - logger.debug(" Contents of %s after replacement", xml_file) - for line in new_xml_data.splitlines(): + logger.debug(" Diff of %s after replacement", xml_file) + for line in unified_diff(xml_data.splitlines(), new_xml_data.splitlines(), + fromfile=xml_file, tofile=xml_file, n=0, lineterm=""): logger.debug(" %s", line) logger.debug("") diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index 0744dc21ad9..9c21c21feeb 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -5,27 +5,25 @@ """ # pylint: disable=too-many-lines import contextlib -from datetime import datetime -from getpass import getuser import json -from logging import getLogger import os import re import signal import time +from datetime import datetime +from getpass import getuser +from logging import getLogger from avocado.utils import process from ClusterShell.NodeSet import NodeSet - -from command_utils_base import \ - BasicParameter, CommandWithParameters, EnvironmentVariables, LogParameter, ObjectWithParameters +from command_utils_base import (BasicParameter, CommandWithParameters, EnvironmentVariables, + LogParameter, ObjectWithParameters) from exception_utils import CommandFailure -from general_utils import check_file_exists, \ - run_command, DaosTestError, get_job_manager_class, create_directory, \ - distribute_files, change_file_owner, get_file_listing, run_pcmd, \ - get_subprocess_stdout -from user_utils import get_primary_group +from general_utils import (DaosTestError, change_file_owner, check_file_exists, create_directory, + distribute_files, get_file_listing, get_job_manager_class, + get_subprocess_stdout, run_command, run_pcmd) from run_utils import command_as_user +from user_utils import get_primary_group from yaml_utils import get_yaml_data diff --git a/src/tests/ftest/util/command_utils_base.py b/src/tests/ftest/util/command_utils_base.py index 5f3d4585192..07672cbdf5f 100644 --- a/src/tests/ftest/util/command_utils_base.py +++ b/src/tests/ftest/util/command_utils_base.py @@ -3,8 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from logging import getLogger import os +from logging import getLogger + import yaml from exception_utils import CommandFailure diff --git a/src/tests/ftest/util/configuration_utils.py b/src/tests/ftest/util/configuration_utils.py index 928caa71264..689ac8456c4 100644 --- a/src/tests/ftest/util/configuration_utils.py +++ b/src/tests/ftest/util/configuration_utils.py @@ -3,13 +3,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from logging import getLogger import re +from logging import getLogger from command_utils_base import BasicParameter, ObjectWithParameters from general_utils import get_host_data - DATA_ERROR = "[ERROR]" diff --git a/src/tests/ftest/util/cont_security_test_base.py b/src/tests/ftest/util/cont_security_test_base.py index 6bfb1ef9801..ce9c3bf93a5 100644 --- a/src/tests/ftest/util/cont_security_test_base.py +++ b/src/tests/ftest/util/cont_security_test_base.py @@ -3,15 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import grp import os import pwd -import grp import re -from avocado.core.exceptions import TestFail - -from apricot import TestWithServers import general_utils +from apricot import TestWithServers +from avocado.core.exceptions import TestFail from general_utils import DaosTestError from security_test_base import acl_entry diff --git a/src/tests/ftest/util/container_rf_test_base.py b/src/tests/ftest/util/container_rf_test_base.py index feb6adc4e06..53fcbe79319 100644 --- a/src/tests/ftest/util/container_rf_test_base.py +++ b/src/tests/ftest/util/container_rf_test_base.py @@ -5,9 +5,9 @@ """ import re -from rebuild_test_base import RebuildTestBase -from general_utils import DaosTestError from daos_utils import DaosCommand +from general_utils import DaosTestError +from rebuild_test_base import RebuildTestBase class ContRedundancyFactor(RebuildTestBase): diff --git a/src/tests/ftest/util/control_test_base.py b/src/tests/ftest/util/control_test_base.py index c99d954e5a9..eff064f53f2 100644 --- a/src/tests/ftest/util/control_test_base.py +++ b/src/tests/ftest/util/control_test_base.py @@ -4,9 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers +from ClusterShell.NodeSet import NodeSet class ControlTestBase(TestWithServers): diff --git a/src/tests/ftest/util/daos_core_base.py b/src/tests/ftest/util/daos_core_base.py index cc560e3e0fd..86fbaf3edff 100644 --- a/src/tests/ftest/util/daos_core_base.py +++ b/src/tests/ftest/util/daos_core_base.py @@ -6,12 +6,12 @@ import os import shutil -from avocado import fail_on from apricot import TestWithServers -from general_utils import get_log_file +from avocado import fail_on from cmocka_utils import CmockaUtils from exception_utils import CommandFailure +from general_utils import get_log_file from job_manager_utils import get_job_manager from test_utils_pool import POOL_TIMEOUT_INCREMENT diff --git a/src/tests/ftest/util/daos_io_conf.py b/src/tests/ftest/util/daos_io_conf.py index 4e5dab0dc3e..0f37479b8e8 100644 --- a/src/tests/ftest/util/daos_io_conf.py +++ b/src/tests/ftest/util/daos_io_conf.py @@ -9,8 +9,8 @@ from apricot import TestWithServers from command_utils import ExecutableCommand from command_utils_base import BasicParameter, FormattedParameter -from exception_utils import CommandFailure, MPILoadError from env_modules import load_mpi +from exception_utils import CommandFailure, MPILoadError from job_manager_utils import Orterun diff --git a/src/tests/ftest/util/daos_perf_utils.py b/src/tests/ftest/util/daos_perf_utils.py index b4ad3a87682..c336b73b5c6 100644 --- a/src/tests/ftest/util/daos_perf_utils.py +++ b/src/tests/ftest/util/daos_perf_utils.py @@ -5,8 +5,8 @@ """ import os -from command_utils_base import FormattedParameter from command_utils import ExecutableCommand +from command_utils_base import FormattedParameter class DaosPerfCommand(ExecutableCommand): diff --git a/src/tests/ftest/util/daos_racer_utils.py b/src/tests/ftest/util/daos_racer_utils.py index 5191ddcab8f..a1ebd6e92d6 100644 --- a/src/tests/ftest/util/daos_racer_utils.py +++ b/src/tests/ftest/util/daos_racer_utils.py @@ -6,12 +6,11 @@ import os from ClusterShell.NodeSet import NodeSet - -from command_utils_base import BasicParameter, FormattedParameter -from exception_utils import CommandFailure, MPILoadError from command_utils import ExecutableCommand -from general_utils import pcmd, get_log_file +from command_utils_base import BasicParameter, FormattedParameter from env_modules import load_mpi +from exception_utils import CommandFailure, MPILoadError +from general_utils import get_log_file, pcmd class DaosRacerCommand(ExecutableCommand): diff --git a/src/tests/ftest/util/daos_utils.py b/src/tests/ftest/util/daos_utils.py index 429945f8b00..679ca432414 100644 --- a/src/tests/ftest/util/daos_utils.py +++ b/src/tests/ftest/util/daos_utils.py @@ -7,7 +7,7 @@ import traceback from daos_utils_base import DaosCommandBase -from general_utils import list_to_str, dict_to_str +from general_utils import dict_to_str, list_to_str class DaosCommand(DaosCommandBase): diff --git a/src/tests/ftest/util/daos_utils_base.py b/src/tests/ftest/util/daos_utils_base.py index 7cec7fefece..9130b214dac 100644 --- a/src/tests/ftest/util/daos_utils_base.py +++ b/src/tests/ftest/util/daos_utils_base.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from command_utils_base import FormattedParameter, CommandWithParameters, BasicParameter from command_utils import CommandWithSubCommand +from command_utils_base import BasicParameter, CommandWithParameters, FormattedParameter class DaosCommandBase(CommandWithSubCommand): diff --git a/src/tests/ftest/util/data_mover_test_base.py b/src/tests/ftest/util/data_mover_test_base.py index 88eb2d21577..4a839bb587f 100644 --- a/src/tests/ftest/util/data_mover_test_base.py +++ b/src/tests/ftest/util/data_mover_test_base.py @@ -3,24 +3,22 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import ctypes # pylint: disable=too-many-lines import os -from os.path import join import re -import ctypes - -from pydaos.raw import str_to_c_uuid, DaosContainer, DaosObj, IORequest +from os.path import join +from command_utils_base import BasicParameter, EnvironmentVariables +from data_mover_utils import (ContClone, DcpCommand, DdeserializeCommand, DserializeCommand, + DsyncCommand, FsCopy, uuid_from_obj) +from duns_utils import format_path from exception_utils import CommandFailure -from test_utils_container import TestContainer +from general_utils import create_string_buffer, get_log_file from ior_test_base import IorTestBase from mdtest_test_base import MdtestBase -from data_mover_utils import DcpCommand, DsyncCommand, FsCopy, ContClone -from data_mover_utils import DserializeCommand, DdeserializeCommand -from data_mover_utils import uuid_from_obj -from duns_utils import format_path -from general_utils import create_string_buffer, get_log_file -from command_utils_base import BasicParameter, EnvironmentVariables +from pydaos.raw import DaosContainer, DaosObj, IORequest, str_to_c_uuid +from test_utils_container import TestContainer class DataMoverTestBase(IorTestBase, MdtestBase): diff --git a/src/tests/ftest/util/data_mover_utils.py b/src/tests/ftest/util/data_mover_utils.py index d0b1cf17b65..a0b715faa16 100644 --- a/src/tests/ftest/util/data_mover_utils.py +++ b/src/tests/ftest/util/data_mover_utils.py @@ -4,9 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from command_utils_base import FormattedParameter -from command_utils_base import BasicParameter from command_utils import ExecutableCommand +from command_utils_base import BasicParameter, FormattedParameter from job_manager_utils import Mpirun diff --git a/src/tests/ftest/util/dbench_utils.py b/src/tests/ftest/util/dbench_utils.py index 897f77efe13..40313dd6d1c 100644 --- a/src/tests/ftest/util/dbench_utils.py +++ b/src/tests/ftest/util/dbench_utils.py @@ -5,9 +5,8 @@ """ -from command_utils_base import FormattedParameter -from command_utils_base import BasicParameter from command_utils import ExecutableCommand +from command_utils_base import BasicParameter, FormattedParameter from job_manager_utils import Mpirun diff --git a/src/tests/ftest/util/dfuse_test_base.py b/src/tests/ftest/util/dfuse_test_base.py index 16d5e9aeb1c..df1f6c8d721 100644 --- a/src/tests/ftest/util/dfuse_test_base.py +++ b/src/tests/ftest/util/dfuse_test_base.py @@ -4,10 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from agent_utils import include_local_host - from apricot import TestWithServers -from exception_utils import CommandFailure from dfuse_utils import get_dfuse, start_dfuse +from exception_utils import CommandFailure class DfuseTestBase(TestWithServers): diff --git a/src/tests/ftest/util/dfuse_utils.py b/src/tests/ftest/util/dfuse_utils.py index 17c67f7be79..afea1ae187b 100644 --- a/src/tests/ftest/util/dfuse_utils.py +++ b/src/tests/ftest/util/dfuse_utils.py @@ -4,15 +4,15 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import os -from ClusterShell.NodeSet import NodeSet +import time -from command_utils_base import FormattedParameter, BasicParameter -from exception_utils import CommandFailure +from ClusterShell.NodeSet import NodeSet from command_utils import ExecutableCommand +from command_utils_base import BasicParameter, FormattedParameter +from exception_utils import CommandFailure from general_utils import check_file_exists, get_log_file -from run_utils import run_remote, command_as_user +from run_utils import command_as_user, run_remote class DfuseCommand(ExecutableCommand): diff --git a/src/tests/ftest/util/distro_utils.py b/src/tests/ftest/util/distro_utils.py index 90565365853..78d26e651f1 100644 --- a/src/tests/ftest/util/distro_utils.py +++ b/src/tests/ftest/util/distro_utils.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import re + # pylint: disable-next=wildcard-import, unused-wildcard-import from avocado.utils.distro import * # noqa: F403 diff --git a/src/tests/ftest/util/dmg_utils.py b/src/tests/ftest/util/dmg_utils.py index 43192a6c7aa..b1680d17e46 100644 --- a/src/tests/ftest/util/dmg_utils.py +++ b/src/tests/ftest/util/dmg_utils.py @@ -3,16 +3,16 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import re +from grp import getgrgid # pylint: disable=too-many-lines from logging import getLogger -from grp import getgrgid from pwd import getpwuid -import re -from exception_utils import CommandFailure from dmg_utils_base import DmgCommandBase -from general_utils import get_numeric_list, dict_to_str -from dmg_utils_params import DmgYamlParameters, DmgTransportCredentials +from dmg_utils_params import DmgTransportCredentials, DmgYamlParameters +from exception_utils import CommandFailure +from general_utils import dict_to_str, get_numeric_list class DmgJsonCommandFailure(CommandFailure): diff --git a/src/tests/ftest/util/dmg_utils_base.py b/src/tests/ftest/util/dmg_utils_base.py index b563f8ec622..78c685eb3dd 100644 --- a/src/tests/ftest/util/dmg_utils_base.py +++ b/src/tests/ftest/util/dmg_utils_base.py @@ -6,9 +6,8 @@ from socket import gethostname from ClusterShell.NodeSet import NodeSet - -from command_utils_base import FormattedParameter, CommandWithParameters, BasicParameter from command_utils import CommandWithSubCommand, YamlCommand +from command_utils_base import BasicParameter, CommandWithParameters, FormattedParameter from general_utils import nodeset_append_suffix diff --git a/src/tests/ftest/util/dmg_utils_params.py b/src/tests/ftest/util/dmg_utils_params.py index 248b398d163..ff9048a3b62 100644 --- a/src/tests/ftest/util/dmg_utils_params.py +++ b/src/tests/ftest/util/dmg_utils_params.py @@ -4,8 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from command_utils_base import \ - BasicParameter, LogParameter, YamlParameters, TransportCredentials +from command_utils_base import BasicParameter, LogParameter, TransportCredentials, YamlParameters class DmgTransportCredentials(TransportCredentials): diff --git a/src/tests/ftest/util/ec_utils.py b/src/tests/ftest/util/ec_utils.py index df50dc49881..d706b32320f 100644 --- a/src/tests/ftest/util/ec_utils.py +++ b/src/tests/ftest/util/ec_utils.py @@ -3,20 +3,19 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import queue import re import threading -import queue import time -from pydaos.raw import DaosApiError - -from nvme_utils import ServerFillUp -from daos_utils import DaosCommand from apricot import TestWithServers -from mdtest_test_base import MdtestBase -from fio_test_base import FioBase +from daos_utils import DaosCommand from exception_utils import CommandFailure +from fio_test_base import FioBase from general_utils import DaosTestError, run_pcmd +from mdtest_test_base import MdtestBase +from nvme_utils import ServerFillUp +from pydaos.raw import DaosApiError def get_data_parity_number(log, oclass): diff --git a/src/tests/ftest/util/exception_utils.py b/src/tests/ftest/util/exception_utils.py index 3f1be6dfdc3..d50b73a0167 100644 --- a/src/tests/ftest/util/exception_utils.py +++ b/src/tests/ftest/util/exception_utils.py @@ -5,7 +5,8 @@ """ import os -from env_modules import show_avail, get_module_list + +from env_modules import get_module_list, show_avail from general_utils import run_command diff --git a/src/tests/ftest/util/fault_config_utils.py b/src/tests/ftest/util/fault_config_utils.py index e0f6b4e37e3..d398997320e 100644 --- a/src/tests/ftest/util/fault_config_utils.py +++ b/src/tests/ftest/util/fault_config_utils.py @@ -5,8 +5,9 @@ """ import os + import yaml -from general_utils import distribute_files, run_command, DaosTestError +from general_utils import DaosTestError, distribute_files, run_command from run_utils import get_clush_command # a lookup table of predefined faults diff --git a/src/tests/ftest/util/file_count_test_base.py b/src/tests/ftest/util/file_count_test_base.py index 90793fc8ec2..88e1ceb1f27 100644 --- a/src/tests/ftest/util/file_count_test_base.py +++ b/src/tests/ftest/util/file_count_test_base.py @@ -4,7 +4,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from avocado.core.exceptions import TestFail - from ior_test_base import IorTestBase from mdtest_test_base import MdtestBase from oclass_utils import extract_redundancy_factor diff --git a/src/tests/ftest/util/fio_utils.py b/src/tests/ftest/util/fio_utils.py index 6c8e1bc31aa..7a0fa3a60ef 100644 --- a/src/tests/ftest/util/fio_utils.py +++ b/src/tests/ftest/util/fio_utils.py @@ -4,11 +4,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from ClusterShell.NodeSet import NodeSet - -from run_utils import run_remote -from command_utils_base import BasicParameter, FormattedParameter, CommandWithParameters -from exception_utils import CommandFailure from command_utils import ExecutableCommand +from command_utils_base import BasicParameter, CommandWithParameters, FormattedParameter +from exception_utils import CommandFailure +from run_utils import run_remote class FioCommand(ExecutableCommand): diff --git a/src/tests/ftest/util/general_utils.py b/src/tests/ftest/util/general_utils.py index 7622853c148..754d80c7411 100644 --- a/src/tests/ftest/util/general_utils.py +++ b/src/tests/ftest/util/general_utils.py @@ -5,27 +5,26 @@ """ # pylint: disable=too-many-lines -from logging import getLogger +import ctypes +import math import os -import re import random +import re import string import time -import ctypes -import math +from datetime import datetime from getpass import getuser from importlib import import_module +from logging import getLogger from socket import gethostname -from datetime import datetime from avocado.core.settings import settings from avocado.core.version import MAJOR from avocado.utils import process -from ClusterShell.Task import task_self from ClusterShell.NodeSet import NodeSet - +from ClusterShell.Task import task_self +from run_utils import RunException, get_clush_command, run_local, run_remote from user_utils import get_chown_command, get_primary_group -from run_utils import get_clush_command, run_remote, run_local, RunException class DaosTestError(Exception): diff --git a/src/tests/ftest/util/io_utilities.py b/src/tests/ftest/util/io_utilities.py index 307ac94cb23..125248c8e20 100644 --- a/src/tests/ftest/util/io_utilities.py +++ b/src/tests/ftest/util/io_utilities.py @@ -3,17 +3,16 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from logging import getLogger import os import random import shutil import tempfile import time +from logging import getLogger +from general_utils import DaosTestError, get_random_bytes from pydaos.raw import DaosApiError -from general_utils import get_random_bytes, DaosTestError - class DirTree(): """ diff --git a/src/tests/ftest/util/ior_intercept_test_base.py b/src/tests/ftest/util/ior_intercept_test_base.py index 7cec0fd32db..fb28c7356ca 100644 --- a/src/tests/ftest/util/ior_intercept_test_base.py +++ b/src/tests/ftest/util/ior_intercept_test_base.py @@ -5,9 +5,10 @@ """ import os + +from general_utils import percent_change from ior_test_base import IorTestBase from ior_utils import IorCommand, IorMetrics -from general_utils import percent_change class IorInterceptTestBase(IorTestBase): diff --git a/src/tests/ftest/util/ior_test_base.py b/src/tests/ftest/util/ior_test_base.py index 0243001fa1f..08c65a41754 100644 --- a/src/tests/ftest/util/ior_test_base.py +++ b/src/tests/ftest/util/ior_test_base.py @@ -6,12 +6,11 @@ import os from ClusterShell.NodeSet import NodeSet - from dfuse_test_base import DfuseTestBase -from ior_utils import IorCommand from exception_utils import CommandFailure +from general_utils import get_random_string, pcmd +from ior_utils import IorCommand from job_manager_utils import get_job_manager -from general_utils import pcmd, get_random_string class IorTestBase(DfuseTestBase): diff --git a/src/tests/ftest/util/ior_utils.py b/src/tests/ftest/util/ior_utils.py index 8f8c35880d0..cd54b0e19af 100644 --- a/src/tests/ftest/util/ior_utils.py +++ b/src/tests/ftest/util/ior_utils.py @@ -8,11 +8,11 @@ from enum import IntEnum from avocado.utils.process import CmdResult -from command_utils_base import FormattedParameter, BasicParameter -from exception_utils import CommandFailure from command_utils import SubProcessCommand -from general_utils import get_log_file +from command_utils_base import BasicParameter, FormattedParameter from duns_utils import format_path +from exception_utils import CommandFailure +from general_utils import get_log_file def get_ior(test, manager, hosts, path, slots, namespace="/run/ior/*", ior_params=None): diff --git a/src/tests/ftest/util/job_manager_utils.py b/src/tests/ftest/util/job_manager_utils.py index 2d61159cedc..adbf004162c 100644 --- a/src/tests/ftest/util/job_manager_utils.py +++ b/src/tests/ftest/util/job_manager_utils.py @@ -7,17 +7,15 @@ import re import time # pylint: disable=too-many-lines -from distutils.spawn import \ - find_executable # pylint: disable=deprecated-module +from distutils.spawn import find_executable # pylint: disable=deprecated-module from ClusterShell.NodeSet import NodeSet from command_utils import ExecutableCommand, SystemctlCommand -from command_utils_base import (BasicParameter, EnvironmentVariables, - FormattedParameter) +from command_utils_base import BasicParameter, EnvironmentVariables, FormattedParameter from env_modules import load_mpi from exception_utils import CommandFailure, MPILoadError -from general_utils import (get_job_manager_class, get_journalctl_command, - journalctl_time, pcmd, run_pcmd) +from general_utils import (get_job_manager_class, get_journalctl_command, journalctl_time, pcmd, + run_pcmd) from run_utils import run_remote, stop_processes from write_host_file import write_host_file diff --git a/src/tests/ftest/util/macsio_test_base.py b/src/tests/ftest/util/macsio_test_base.py index f5565b8aaab..e7519a51eb3 100644 --- a/src/tests/ftest/util/macsio_test_base.py +++ b/src/tests/ftest/util/macsio_test_base.py @@ -5,8 +5,8 @@ """ from apricot import TestWithServers from exception_utils import CommandFailure -from macsio_util import MacsioCommand from general_utils import get_log_file +from macsio_util import MacsioCommand class MacsioTestBase(TestWithServers): diff --git a/src/tests/ftest/util/macsio_util.py b/src/tests/ftest/util/macsio_util.py index de95ac7081d..cf64dbcb030 100644 --- a/src/tests/ftest/util/macsio_util.py +++ b/src/tests/ftest/util/macsio_util.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from command_utils_base import FormattedParameter from command_utils import ExecutableCommand +from command_utils_base import FormattedParameter from general_utils import get_log_file, pcmd diff --git a/src/tests/ftest/util/mdtest_test_base.py b/src/tests/ftest/util/mdtest_test_base.py index bf39b97f2fa..8646d4ef0ac 100644 --- a/src/tests/ftest/util/mdtest_test_base.py +++ b/src/tests/ftest/util/mdtest_test_base.py @@ -5,10 +5,11 @@ """ import os + from dfuse_test_base import DfuseTestBase -from mdtest_utils import MdtestCommand from exception_utils import CommandFailure from job_manager_utils import get_job_manager +from mdtest_utils import MdtestCommand class MdtestBase(DfuseTestBase): diff --git a/src/tests/ftest/util/mdtest_utils.py b/src/tests/ftest/util/mdtest_utils.py index d54f4a605cc..4c6d16b29a1 100644 --- a/src/tests/ftest/util/mdtest_utils.py +++ b/src/tests/ftest/util/mdtest_utils.py @@ -7,8 +7,8 @@ import os import re -from command_utils_base import FormattedParameter from command_utils import ExecutableCommand +from command_utils_base import FormattedParameter from general_utils import get_log_file diff --git a/src/tests/ftest/util/mpiio_test_base.py b/src/tests/ftest/util/mpiio_test_base.py index e83887c0382..b3fd058de13 100644 --- a/src/tests/ftest/util/mpiio_test_base.py +++ b/src/tests/ftest/util/mpiio_test_base.py @@ -7,11 +7,10 @@ import os from apricot import TestWithServers - from command_utils_base import CommandFailure, EnvironmentVariables -from job_manager_utils import get_job_manager -from mpiio_utils import LLNLCommand, Mpi4pyCommand, RomioCommand, Hdf5Command from duns_utils import format_path +from job_manager_utils import get_job_manager +from mpiio_utils import Hdf5Command, LLNLCommand, Mpi4pyCommand, RomioCommand class MpiioTests(TestWithServers): diff --git a/src/tests/ftest/util/mpiio_utils.py b/src/tests/ftest/util/mpiio_utils.py index 2c596d38643..44b40a98551 100644 --- a/src/tests/ftest/util/mpiio_utils.py +++ b/src/tests/ftest/util/mpiio_utils.py @@ -4,8 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from sys import version_info -from command_utils_base import FormattedParameter + from command_utils import ExecutableCommand +from command_utils_base import FormattedParameter class LLNLCommand(ExecutableCommand): diff --git a/src/tests/ftest/util/nvme_utils.py b/src/tests/ftest/util/nvme_utils.py index 3ba6fe2ec53..e92128b3d10 100644 --- a/src/tests/ftest/util/nvme_utils.py +++ b/src/tests/ftest/util/nvme_utils.py @@ -3,14 +3,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import threading import re +import threading import time from avocado import fail_on from avocado.core.exceptions import TestFail - -from dmg_utils import get_storage_query_device_uuids, get_dmg_response +from dmg_utils import get_dmg_response, get_storage_query_device_uuids from exception_utils import CommandFailure from ior_test_base import IorTestBase from ior_utils import IorCommand diff --git a/src/tests/ftest/util/osa_utils.py b/src/tests/ftest/util/osa_utils.py index 493fcf189a7..0b9c9002809 100644 --- a/src/tests/ftest/util/osa_utils.py +++ b/src/tests/ftest/util/osa_utils.py @@ -5,17 +5,16 @@ """ import ctypes import queue -import time -import threading import re - -from pydaos.raw import DaosContainer, IORequest, DaosObj, DaosApiError +import threading +import time from avocado import fail_on -from ior_test_base import IorTestBase -from mdtest_test_base import MdtestBase from exception_utils import CommandFailure from general_utils import create_string_buffer, run_command +from ior_test_base import IorTestBase +from mdtest_test_base import MdtestBase +from pydaos.raw import DaosApiError, DaosContainer, DaosObj, IORequest class OSAUtils(MdtestBase, IorTestBase): @@ -457,8 +456,11 @@ def ior_thread(self, pool, oclass, test, flags, single_cont_read=True, fail_on_w "Detected container redundancy factor: %s", self.container.properties.value) self.ior_cmd.dfs_oclass.update(None, "ior.dfs_oclass") self.ior_cmd.dfs_dir_oclass.update(None, "ior.dfs_dir_oclass") + # Run run_ior_with_pool without invoking the pool query method for + # displaying pool space information (display_space=False) self.run_ior_with_pool(create_pool=False, create_cont=False, fail_on_warning=fail_on_warning, + display_space=False, out_queue=self.out_queue) if fail_on_warning and not self.out_queue.empty(): self.assert_on_exception() diff --git a/src/tests/ftest/util/performance_test_base.py b/src/tests/ftest/util/performance_test_base.py index 4e9ef61004d..bf7a24907ef 100644 --- a/src/tests/ftest/util/performance_test_base.py +++ b/src/tests/ftest/util/performance_test_base.py @@ -6,15 +6,14 @@ import os import time +import oclass_utils from avocado.core.exceptions import TestFail - +from exception_utils import CommandFailure +from general_utils import get_subprocess_stdout from ior_test_base import IorTestBase +from ior_utils import IorMetrics from mdtest_test_base import MdtestBase from mdtest_utils import MdtestMetrics -from general_utils import get_subprocess_stdout -from ior_utils import IorMetrics -import oclass_utils -from exception_utils import CommandFailure class PerformanceTestBase(IorTestBase, MdtestBase): diff --git a/src/tests/ftest/util/pool_create_all_base.py b/src/tests/ftest/util/pool_create_all_base.py index 2ab16d42485..81b7b4b079b 100644 --- a/src/tests/ftest/util/pool_create_all_base.py +++ b/src/tests/ftest/util/pool_create_all_base.py @@ -5,9 +5,8 @@ """ import sys -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail from general_utils import bytes_to_human diff --git a/src/tests/ftest/util/pool_security_test_base.py b/src/tests/ftest/util/pool_security_test_base.py index 4dc4250af77..c861248f17a 100644 --- a/src/tests/ftest/util/pool_security_test_base.py +++ b/src/tests/ftest/util/pool_security_test_base.py @@ -3,14 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import grp import os import random -import grp import re -from apricot import TestWithServers import agent_utils as agu import security_test_base as secTestBase +from apricot import TestWithServers PERMISSIONS = ["", "r", "w", "rw"] DENY_ACCESS = "-1001" diff --git a/src/tests/ftest/util/rebuild_test_base.py b/src/tests/ftest/util/rebuild_test_base.py index 1435aa1815e..bdb31ea9807 100644 --- a/src/tests/ftest/util/rebuild_test_base.py +++ b/src/tests/ftest/util/rebuild_test_base.py @@ -4,7 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from apricot import TestWithServers -from command_utils_base import ObjectWithParameters, BasicParameter +from command_utils_base import BasicParameter, ObjectWithParameters from daos_utils import DaosCommand diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index 730dbb3bbd1..b382a893522 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -3,10 +3,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from socket import gethostname -import subprocess # nosec import shlex +import subprocess # nosec import time +from socket import gethostname + from ClusterShell.NodeSet import NodeSet from ClusterShell.Task import task_self diff --git a/src/tests/ftest/util/scrubber_test_base.py b/src/tests/ftest/util/scrubber_test_base.py index 95a2eb770a8..7e938662e9f 100644 --- a/src/tests/ftest/util/scrubber_test_base.py +++ b/src/tests/ftest/util/scrubber_test_base.py @@ -5,8 +5,8 @@ """ import threading -from scrubber_utils import ScrubberUtils from ior_test_base import IorTestBase +from scrubber_utils import ScrubberUtils class TestWithScrubber(IorTestBase): diff --git a/src/tests/ftest/util/security_test_base.py b/src/tests/ftest/util/security_test_base.py index 85c8ad2e050..a0b83426777 100644 --- a/src/tests/ftest/util/security_test_base.py +++ b/src/tests/ftest/util/security_test_base.py @@ -6,6 +6,7 @@ import os import random + from general_utils import pcmd diff --git a/src/tests/ftest/util/server_utils.py b/src/tests/ftest/util/server_utils.py index c8c36862c0f..8f20bf6c783 100644 --- a/src/tests/ftest/util/server_utils.py +++ b/src/tests/ftest/util/server_utils.py @@ -5,25 +5,24 @@ """ # pylint: disable=too-many-lines -from collections import defaultdict -from getpass import getuser import os +import random import re import time -import random +from collections import defaultdict +from getpass import getuser from avocado import fail_on - -from command_utils_base import CommonConfig, BasicParameter from command_utils import SubprocessManager +from command_utils_base import BasicParameter, CommonConfig from dmg_utils import get_dmg_command from exception_utils import CommandFailure -from general_utils import pcmd, get_log_file, list_to_str, get_display_size, run_pcmd +from general_utils import get_display_size, get_log_file, list_to_str, pcmd, run_pcmd from host_utils import get_local_host -from server_utils_base import ServerFailed, DaosServerCommand, DaosServerInformation +from run_utils import run_remote, stop_processes +from server_utils_base import DaosServerCommand, DaosServerInformation, ServerFailed from server_utils_params import DaosServerTransportCredentials, DaosServerYamlParameters from user_utils import get_chown_command -from run_utils import run_remote, stop_processes def get_server_command(group, cert_dir, bin_dir, config_file, config_temp=None): diff --git a/src/tests/ftest/util/server_utils_base.py b/src/tests/ftest/util/server_utils_base.py index ab68914b24c..068c62919f0 100644 --- a/src/tests/ftest/util/server_utils_base.py +++ b/src/tests/ftest/util/server_utils_base.py @@ -3,14 +3,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from logging import getLogger import os import re +from logging import getLogger from ClusterShell.NodeSet import NodeSet - -from command_utils_base import FormattedParameter, CommandWithParameters -from command_utils import YamlCommand, CommandWithSubCommand +from command_utils import CommandWithSubCommand, YamlCommand +from command_utils_base import CommandWithParameters, FormattedParameter from data_utils import dict_extract_values, list_flatten from dmg_utils import get_dmg_response from exception_utils import CommandFailure diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index 8f456706c04..a79c750b54d 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -5,8 +5,7 @@ """ import os -from command_utils_base import \ - BasicParameter, LogParameter, YamlParameters, TransportCredentials +from command_utils_base import BasicParameter, LogParameter, TransportCredentials, YamlParameters MAX_STORAGE_TIERS = 5 @@ -108,7 +107,7 @@ def __init__(self, filename, common_yaml): # is set for the running process. If group look up fails or user # is not member, use uid return from user lookup. # - default_provider = os.environ.get("CRT_PHY_ADDR_STR", "ofi+sockets") + default_provider = os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp;ofi_rxm") # All log files should be placed in the same directory on each host to # enable easy log file archiving by launch.py diff --git a/src/tests/ftest/util/slurm_utils.py b/src/tests/ftest/util/slurm_utils.py index d43208e3d61..9ba92b0f8c9 100644 --- a/src/tests/ftest/util/slurm_utils.py +++ b/src/tests/ftest/util/slurm_utils.py @@ -5,9 +5,9 @@ """ import os import random -import time -import threading import re +import threading +import time from ClusterShell.NodeSet import NodeSet, NodeSetParseError # pylint: disable=import-error,no-name-in-module diff --git a/src/tests/ftest/util/soak_test_base.py b/src/tests/ftest/util/soak_test_base.py index dc0e1912bf3..03c01106894 100644 --- a/src/tests/ftest/util/soak_test_base.py +++ b/src/tests/ftest/util/soak_test_base.py @@ -4,33 +4,32 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import multiprocessing import os +import random +import socket +import threading import time from datetime import datetime, timedelta -import multiprocessing -import threading -import random from filecmp import cmp from getpass import getuser -import socket +import slurm_utils +from agent_utils import include_local_host from apricot import TestWithServers from ClusterShell.NodeSet import NodeSet -from agent_utils import include_local_host +from dmg_utils import DmgCommand from exception_utils import CommandFailure from general_utils import journalctl_time from host_utils import get_local_host -import slurm_utils -from dmg_utils import DmgCommand -from run_utils import run_local, run_remote, RunException -from soak_utils import ddhhmmss_format, add_pools, \ - launch_snapshot, launch_exclude_reintegrate, launch_extend, \ - create_ior_cmdline, cleanup_dfuse, create_fio_cmdline, \ - build_job_script, SoakTestError, launch_server_stop_start, get_harassers, \ - create_racer_cmdline, run_event_check, run_monitor_check, \ - create_mdtest_cmdline, reserved_file_copy, run_metrics_check, \ - get_journalctl, get_daos_server_logs, create_macsio_cmdline, \ - create_app_cmdline, create_dm_cmdline, launch_vmd_identify_check +from run_utils import RunException, run_local, run_remote +from soak_utils import (SoakTestError, add_pools, build_job_script, cleanup_dfuse, + create_app_cmdline, create_dm_cmdline, create_fio_cmdline, + create_ior_cmdline, create_macsio_cmdline, create_mdtest_cmdline, + create_racer_cmdline, ddhhmmss_format, get_daos_server_logs, get_harassers, + get_journalctl, launch_exclude_reintegrate, launch_extend, + launch_server_stop_start, launch_snapshot, launch_vmd_identify_check, + reserved_file_copy, run_event_check, run_metrics_check, run_monitor_check) class SoakTestBase(TestWithServers): diff --git a/src/tests/ftest/util/soak_utils.py b/src/tests/ftest/util/soak_utils.py index 577fde705e3..9aaec2fdbf1 100644 --- a/src/tests/ftest/util/soak_utils.py +++ b/src/tests/ftest/util/soak_utils.py @@ -6,33 +6,31 @@ # pylint: disable=too-many-lines import os -import time import random -import threading import re +import threading +import time from itertools import product +import slurm_utils from avocado.core.exceptions import TestFail -from pydaos.raw import DaosSnapshot, DaosApiError - -from ior_utils import IorCommand -from fio_utils import FioCommand -from mdtest_utils import MdtestCommand +from command_utils_base import EnvironmentVariables from daos_racer_utils import DaosRacerCommand from data_mover_utils import DcpCommand, FsCopy from dfuse_utils import Dfuse from dmg_utils import get_storage_query_device_info +from duns_utils import format_path +from fio_utils import FioCommand +from general_utils import (DaosTestError, get_host_data, get_log_file, get_random_bytes, + get_random_string, list_to_str, pcmd, run_command, run_pcmd) +from ior_utils import IorCommand from job_manager_utils import Mpirun -from general_utils import get_host_data, get_random_string, \ - run_command, DaosTestError, pcmd, get_random_bytes, \ - run_pcmd, list_to_str, get_log_file -from command_utils_base import EnvironmentVariables -import slurm_utils -from run_utils import run_remote -from test_utils_container import TestContainer from macsio_util import MacsioCommand +from mdtest_utils import MdtestCommand from oclass_utils import extract_redundancy_factor -from duns_utils import format_path +from pydaos.raw import DaosApiError, DaosSnapshot +from run_utils import run_remote +from test_utils_container import TestContainer H_LOCK = threading.Lock() diff --git a/src/tests/ftest/util/support_test_base.py b/src/tests/ftest/util/support_test_base.py new file mode 100644 index 00000000000..27b4a20e893 --- /dev/null +++ b/src/tests/ftest/util/support_test_base.py @@ -0,0 +1,139 @@ +""" +(C) Copyright 2023 Intel Corporation. + +SPDX-License-Identifier: BSD-2-Clause-Patent +""" +import os +from datetime import datetime + +from control_test_base import ControlTestBase +from run_utils import command_as_user, run_remote + + +class SupportTestBase(ControlTestBase): + """Class for Support log collection """ + + def __init__(self, *args, **kwargs): + """Initialize a SupportTestBase object.""" + super().__init__(*args, **kwargs) + self.target_folder = None + self.custom_log_dir = None + self.custom_log_file = None + self.custom_log_data = None + self.log_hosts = None + self.run_user = 'root' + self.extract_dir = os.path.join(self.base_test_dir, "extracted_support_logs") + + def create_custom_log(self, folder_name): + """Create custom log directory with custom data file on each servers. + + Args: + folder_name (str): Name of the custom folder + """ + server_custom_log = self.base_test_dir + self.custom_log_dir = os.path.join(server_custom_log, folder_name) + self.custom_log_file = os.path.join(self.custom_log_dir, "Custom_File") + self.target_folder = os.path.join(self.base_test_dir, "DAOS_Support") + + # make the custom log dir on node (clients or servers) + mkdir_cmd = "mkdir -p {}".format(self.custom_log_dir) + result = run_remote(self.log, self.log_hosts, mkdir_cmd) + if not result.passed: + self.fail("Failed to create the custom log dir {} ".format(result)) + + # Get date-time object containing current date and time + now = datetime.now() + self.custom_log_data = now.strftime("%d/%m/%Y %H:%M:%S") + + # Create the custom log file on node (clients or servers) + create_file = " echo \'{}\' > {}".format(self.custom_log_data, self.custom_log_file) + result = run_remote(self.log, self.log_hosts, create_file) + if not result.passed: + self.fail("Failed to create the custom log file {} ".format(result)) + + def verify_custom_log_data(self): + """Verify custom log files is collected and part of archive. + + """ + getfilename = "find {} -name {}".format( + self.extract_dir, os.path.basename(self.custom_log_file)) + findcmd = command_as_user(getfilename, self.run_user) + readfiledata = command_as_user("xargs cat", self.run_user) + + result = run_remote(self.log, self.log_hosts, findcmd + "|" + readfiledata) + if not result.passed: + self.fail("Failed to read the custom log file {} ".format(result)) + + for output in result.all_stdout.values(): + if self.custom_log_data not in output: + self.fail("Expected custom_log_data {} not found in log file {}" + .format(self.custom_log_data, self.custom_log_file)) + + def extract_logs(self, tar_gz_filename): + """Extract the logs files which are in collected archive. + + Args: + tar_gz_filename (str): Log archive File name + + Raises: + Test Failure: If unable to run any command. + + """ + # Create the new extract directory + command = command_as_user("mkdir -p {}".format(self.extract_dir), self.run_user) + result = run_remote(self.log, self.log_hosts, command) + if not result.passed: + self.fail("cmd {} failed, result:{}".format(command, result)) + + # Extract The tar.gz file to newly created directory + command = command_as_user("tar -xf {} -C {}".format(tar_gz_filename, self.extract_dir), + self.run_user) + result = run_remote(self.log, self.log_hosts, command) + if not result.passed: + self.fail("Failed to extract the {} file, result:{}".format(tar_gz_filename, result)) + + def validate_server_log_files(self): + """Verify all the server logs files are collected and part of archive. + + Raises: + Test Failure: If unable to run any command. + + """ + log_files = [] + helper_log_file = self.server_managers[0].get_config_value("helper_log_file") + log_files.append(os.path.basename(helper_log_file)) + control_log_file = self.server_managers[0].get_config_value("control_log_file") + log_files.append(os.path.basename(control_log_file)) + log_files.append(self.params.get("log_file", "/run/server_config/engines/0/*")) + log_files.append(self.params.get("log_file", "/run/server_config/engines/1/*")) + + # Verify server log files are collected. + for log_file in log_files: + list_file = "ls -lsaRt {} | grep {}".format(self.extract_dir, log_file) + command = command_as_user(list_file, self.run_user) + result = run_remote(self.log, self.log_hosts, command) + if not result.passed: + self.fail("Failed to list the {} file from extracted folder{}".format( + result, self.extract_dir)) + + def cleanup_support_log(self, log_dir): + """ Test cleanup to remove the temporary directory + + Args: + log_dir (str): Name of the log directory to be removed + + Returns: + list: a list of any errors detected when deleting the logs + + """ + error_list = [] + # Remove the log and extract directory + folders = [log_dir, self.extract_dir] + for folder in folders: + delete_cmd = "sudo rm -rf {}*".format(folder) + result = run_remote(self.log, self.log_hosts, delete_cmd) + if not result.passed: + error_list.append( + "Failed to delete folder {} with result:{}".format(folder, result)) + + return error_list diff --git a/src/tests/ftest/util/telemetry_utils.py b/src/tests/ftest/util/telemetry_utils.py index cc1cee34127..6656baa2df3 100644 --- a/src/tests/ftest/util/telemetry_utils.py +++ b/src/tests/ftest/util/telemetry_utils.py @@ -3,8 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from logging import getLogger import re +from logging import getLogger + from ClusterShell.NodeSet import NodeSet @@ -561,7 +562,7 @@ class TelemetryUtils(): "engine_mem_vos_dtx_cmt_ent_48", "engine_mem_vos_vos_obj_360", "engine_mem_vos_vos_lru_size", - "engine_mem_dtx_dtx_leader_handle_336", + "engine_mem_dtx_dtx_leader_handle_344", "engine_mem_dtx_dtx_entry_40"] ENGINE_MEM_TOTAL_USAGE_METRICS = [ "engine_mem_total_mem"] diff --git a/src/tests/ftest/util/test_utils_base.py b/src/tests/ftest/util/test_utils_base.py index 49562802c53..af032a5160c 100644 --- a/src/tests/ftest/util/test_utils_base.py +++ b/src/tests/ftest/util/test_utils_base.py @@ -3,15 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from collections import defaultdict from logging import getLogger -from time import sleep from threading import Lock -from collections import defaultdict +from time import sleep +from command_utils_base import BasicParameter, ObjectWithParameters from pydaos.raw import DaosApiError -from command_utils_base import ObjectWithParameters, BasicParameter - class CallbackHandler(): """Defines a callback method to use with DaosApi class methods.""" diff --git a/src/tests/ftest/util/test_utils_container.py b/src/tests/ftest/util/test_utils_container.py index 6152870ab8f..bc7902dec3a 100644 --- a/src/tests/ftest/util/test_utils_container.py +++ b/src/tests/ftest/util/test_utils_container.py @@ -10,13 +10,11 @@ from time import time from avocado import fail_on -from pydaos.raw import (DaosApiError, DaosContainer, DaosInputParams, - c_uuid_to_str, str_to_c_uuid) - -from test_utils_base import TestDaosApiBase from command_utils_base import BasicParameter from exception_utils import CommandFailure -from general_utils import get_random_bytes, DaosTestError +from general_utils import DaosTestError, get_random_bytes +from pydaos.raw import DaosApiError, DaosContainer, DaosInputParams, c_uuid_to_str, str_to_c_uuid +from test_utils_base import TestDaosApiBase class TestContainerData(): diff --git a/src/tests/ftest/util/test_utils_pool.py b/src/tests/ftest/util/test_utils_pool.py index 37e9ddcfcde..2b4b14f427f 100644 --- a/src/tests/ftest/util/test_utils_pool.py +++ b/src/tests/ftest/util/test_utils_pool.py @@ -3,20 +3,19 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import ctypes +import json # pylint: disable=too-many-lines import os from time import sleep, time -import ctypes -import json -from avocado import fail_on, TestFail -from pydaos.raw import (DaosApiError, DaosPool, c_uuid_to_str, daos_cref) - -from test_utils_base import TestDaosApiBase, LabelGenerator +from avocado import TestFail, fail_on from command_utils import BasicParameter -from exception_utils import CommandFailure -from general_utils import check_pool_files, DaosTestError from dmg_utils import DmgCommand, DmgJsonCommandFailure +from exception_utils import CommandFailure +from general_utils import DaosTestError, check_pool_files +from pydaos.raw import DaosApiError, DaosPool, c_uuid_to_str, daos_cref +from test_utils_base import LabelGenerator, TestDaosApiBase POOL_NAMESPACE = "/run/pool/*" POOL_TIMEOUT_INCREMENT = 200 diff --git a/src/tests/ftest/util/thread_manager.py b/src/tests/ftest/util/thread_manager.py index f67719e8843..6e30c6bdee4 100644 --- a/src/tests/ftest/util/thread_manager.py +++ b/src/tests/ftest/util/thread_manager.py @@ -3,8 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import ThreadPoolExecutor from concurrent.futures import TimeoutError as FuturesTimeoutError +from concurrent.futures import as_completed from logging import getLogger from avocado.utils.process import CmdResult diff --git a/src/tests/ftest/util/vol_test_base.py b/src/tests/ftest/util/vol_test_base.py index 3b532f7219a..b45bcf62082 100644 --- a/src/tests/ftest/util/vol_test_base.py +++ b/src/tests/ftest/util/vol_test_base.py @@ -4,10 +4,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from dfuse_test_base import DfuseTestBase +from command_utils import ExecutableCommand from command_utils_base import EnvironmentVariables +from dfuse_test_base import DfuseTestBase from exception_utils import CommandFailure -from command_utils import ExecutableCommand class VolTestBase(DfuseTestBase): diff --git a/src/tests/ftest/util/write_host_file.py b/src/tests/ftest/util/write_host_file.py index 9f418137a5c..8a6e810c6ba 100644 --- a/src/tests/ftest/util/write_host_file.py +++ b/src/tests/ftest/util/write_host_file.py @@ -3,9 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from logging import getLogger import os import random +from logging import getLogger def write_host_file(hosts, path='/tmp', slots=1): diff --git a/src/tests/ftest/vmd/fault_reintegration.py b/src/tests/ftest/vmd/fault_reintegration.py index 3d8a32e6c79..0380fa9effc 100644 --- a/src/tests/ftest/vmd/fault_reintegration.py +++ b/src/tests/ftest/vmd/fault_reintegration.py @@ -3,17 +3,16 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from multiprocessing import Queue import threading +from multiprocessing import Queue +from apricot import TestWithServers from avocado import fail_on - -from dmg_utils import get_storage_query_device_info, get_dmg_response +from dmg_utils import get_dmg_response, get_storage_query_device_info from exception_utils import CommandFailure from ior_utils import run_ior, thread_run_ior from job_manager_utils import get_job_manager from nvme_utils import set_device_faulty -from apricot import TestWithServers class NvmeFaultReintegrate(TestWithServers): diff --git a/src/tests/ftest/vmd/led.py b/src/tests/ftest/vmd/led.py index f69c808d701..168d1dc29ae 100644 --- a/src/tests/ftest/vmd/led.py +++ b/src/tests/ftest/vmd/led.py @@ -6,7 +6,6 @@ import time from avocado import fail_on - from dmg_utils import get_storage_query_device_uuids from exception_utils import CommandFailure from nvme_utils import set_device_faulty diff --git a/src/tests/suite/SConscript b/src/tests/suite/SConscript index a30f8bc432a..7ca9f0b5148 100644 --- a/src/tests/suite/SConscript +++ b/src/tests/suite/SConscript @@ -1,6 +1,6 @@ """Build test suite""" -import sys import subprocess # nosec +import sys TEST_CMOCKA_SKIP = """ #include diff --git a/src/tests/suite/daos_rebuild_ec.c b/src/tests/suite/daos_rebuild_ec.c index 6669d32490e..072bb9a4692 100644 --- a/src/tests/suite/daos_rebuild_ec.c +++ b/src/tests/suite/daos_rebuild_ec.c @@ -1253,6 +1253,77 @@ rebuild_ec_multiple_failure_tgts(void **state) ioreq_fini(&req); } +static void +rebuild_ec_parity_overwrite_fail_parity(void **state) +{ + test_arg_t *arg = *state; + daos_obj_id_t oid; + struct ioreq req; + char *data; + char *verify_data; + daos_recx_t recx; + d_rank_t data_ranks[2]; + d_rank_t parity_rank; + uint64_t dkey_hash; + int shard_idx; + int stripe_size = 4 * CELL_SIZE; + + if (!test_runable(arg, 8)) + return; + + oid = daos_test_oid_gen(arg->coh, OC_EC_4P2G1, 0, 0, arg->myrank); + ioreq_init(&req, arg->coh, oid, DAOS_IOD_ARRAY, arg); + data = (char *)malloc(stripe_size); + verify_data = (char *)malloc(stripe_size); + make_buffer(data, 'a', stripe_size); + + recx.rx_idx = 0; /* full stripe */ + recx.rx_nr = stripe_size; + insert_recxs("d_key", "a_key", 1, DAOS_TX_NONE, &recx, 1, + data, stripe_size, &req); + + make_buffer(data, 'b', 1000); + memcpy(verify_data, data, stripe_size); + recx.rx_idx = 0; + recx.rx_nr = 1000; + insert_recxs("d_key", "a_key", 1, DAOS_TX_NONE, &recx, 1, + data, 1000, &req); + + dkey_hash = d_hash_murmur64((const unsigned char *)"d_key", strlen("d_key"), 5731); + + /* fail parity epoch */ + shard_idx = (dkey_hash % 6 + 5) % 6; + parity_rank = get_rank_by_oid_shard(arg, oid, shard_idx); + rebuild_single_pool_rank(arg, parity_rank, true); + + /* fail data shard */ + shard_idx = (dkey_hash % 6 + 0) % 6; + data_ranks[0] = get_rank_by_oid_shard(arg, oid, shard_idx); + shard_idx = (dkey_hash % 6 + 1) % 6; + data_ranks[1] = get_rank_by_oid_shard(arg, oid, shard_idx); + + rebuild_pools_ranks(&arg, 1, data_ranks, 2, true); + recx.rx_idx = 0; /* full stripe */ + recx.rx_nr = stripe_size; + lookup_recxs("d_key", "a_key", 1, DAOS_TX_NONE, &recx, 1, + data, stripe_size, &req); + + assert_memory_equal(data, verify_data, stripe_size); + + reintegrate_pools_ranks(&arg, 1, data_ranks, 2, true); + reintegrate_pools_ranks(&arg, 1, &parity_rank, 1, true); + + lookup_recxs("d_key", "a_key", 1, DAOS_TX_NONE, &recx, 1, + data, stripe_size, &req); + + assert_memory_equal(data, verify_data, stripe_size); + + + ioreq_fini(&req); + free(data); + free(verify_data); +} + /** create a new pool/container for each test */ static const struct CMUnitTest rebuild_tests[] = { {"REBUILD0: rebuild partial update with data tgt fail", @@ -1385,6 +1456,9 @@ static const struct CMUnitTest rebuild_tests[] = { {"REBUILD45: multiple shards fail tgts", rebuild_ec_multiple_failure_tgts, rebuild_ec_8nodes_setup, test_teardown}, + {"REBUILD46: fail parity shard and data shards after overwrite", + rebuild_ec_parity_overwrite_fail_parity, rebuild_ec_8nodes_setup, + test_teardown}, }; int diff --git a/src/utils/daos_hdlr.c b/src/utils/daos_hdlr.c index 2da8c3df4d4..1c145bc6b1c 100644 --- a/src/utils/daos_hdlr.c +++ b/src/utils/daos_hdlr.c @@ -1492,10 +1492,8 @@ dm_connect(struct cmd_args_s *ap, dattr.da_props = props; rc = duns_create_path(ca->dst_poh, path, &dattr); if (rc != 0) { - rc = daos_errno2der(rc); - DH_PERROR_DER(ap, rc, "provide a destination pool or UNS path " - "of the form:\n\t --dst | "); - D_GOTO(err, rc); + DH_PERROR_SYS(ap, rc, "failed to create destination UNS path"); + D_GOTO(err, rc = daos_errno2der(rc)); } snprintf(ca->dst_cont, DAOS_PROP_LABEL_MAX_LEN + 1, "%s", dattr.da_cont); } diff --git a/src/vos/lru_array.h b/src/vos/lru_array.h index 193e9cf4260..e1aeeddfd0d 100644 --- a/src/vos/lru_array.h +++ b/src/vos/lru_array.h @@ -334,11 +334,11 @@ lrua_peek_(struct lru_array *array, const uint32_t *idx, void **entryp) * -DER_BUSY Entries need to be evicted to free up * entries in the table */ -#define lrua_allocx(array, idx, key, entryp) \ - lrua_allocx_(array, idx, key, (void **)(entryp)) +#define lrua_allocx(array, idx, key, entryp, stub) \ + lrua_allocx_(array, idx, key, (void **)(entryp), (void **)(stub)) static inline int lrua_allocx_(struct lru_array *array, uint32_t *idx, uint64_t key, - void **entryp) + void **entryp, void **stub) { struct lru_entry *new_entry; int rc; @@ -354,6 +354,8 @@ lrua_allocx_(struct lru_array *array, uint32_t *idx, uint64_t key, return rc; *entryp = new_entry->le_payload; + if (stub != NULL) + *stub = new_entry; return 0; } @@ -378,7 +380,7 @@ lrua_allocx_(struct lru_array *array, uint32_t *idx, uint64_t key, static inline int lrua_alloc_(struct lru_array *array, uint32_t *idx, void **entryp) { - return lrua_allocx_(array, idx, (uint64_t)idx, entryp); + return lrua_allocx_(array, idx, (uint64_t)idx, entryp, NULL); } /** Allocate an entry in place. Used for recreating an old array. @@ -497,4 +499,12 @@ lrua_array_free(struct lru_array *array); void lrua_array_aggregate(struct lru_array *array); +static inline void +lrua_refresh_key(struct lru_entry *entry, uint64_t key) +{ + D_ASSERT(entry != NULL); + + entry->le_key = key; +} + #endif /* __LRU_ARRAY__ */ diff --git a/src/vos/storage_estimator/common/dfs_sb.py b/src/vos/storage_estimator/common/dfs_sb.py index 6aaa07e5c51..6b9f7fa4833 100644 --- a/src/vos/storage_estimator/common/dfs_sb.py +++ b/src/vos/storage_estimator/common/dfs_sb.py @@ -4,11 +4,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import os import ctypes +import os from pydaos.raw import daos_cref -from storage_estimator.vos_structures import ValType, Overhead, AKey, VosValue, DKey, VosObject +from storage_estimator.vos_structures import AKey, DKey, Overhead, ValType, VosObject, VosValue header = '''--- # Sample conflig file DFS files and directories diff --git a/src/vos/storage_estimator/common/explorer.py b/src/vos/storage_estimator/common/explorer.py index 97641419e34..aafc7371989 100644 --- a/src/vos/storage_estimator/common/explorer.py +++ b/src/vos/storage_estimator/common/explorer.py @@ -4,13 +4,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import os import copy +import os import sys -from storage_estimator.vos_structures import VosObject, AKey, DKey, Container, \ - VosValue, Overhead, ValType, KeyType from storage_estimator.util import CommonBase, ObjectClass +from storage_estimator.vos_structures import (AKey, Container, DKey, KeyType, Overhead, ValType, + VosObject, VosValue) class FileInfo(): diff --git a/src/vos/storage_estimator/common/parse_csv.py b/src/vos/storage_estimator/common/parse_csv.py index 3b296a9e990..8820a3e3aaa 100644 --- a/src/vos/storage_estimator/common/parse_csv.py +++ b/src/vos/storage_estimator/common/parse_csv.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from storage_estimator.explorer import AverageFS from storage_estimator.dfs_sb import get_dfs_inode_akey +from storage_estimator.explorer import AverageFS from storage_estimator.util import ProcessBase FILE_SIZES = ['4k', '64k', '128k', '256k', '512k', '768k', '1m', '8m', '64m', diff --git a/src/vos/storage_estimator/common/tests/storage_estimator_test.py b/src/vos/storage_estimator/common/tests/storage_estimator_test.py index 5855995d783..66944e75968 100644 --- a/src/vos/storage_estimator/common/tests/storage_estimator_test.py +++ b/src/vos/storage_estimator/common/tests/storage_estimator_test.py @@ -3,16 +3,17 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import pytest -import unittest -import yaml import os +import unittest -from storage_estimator.vos_structures import VosObject, AKey, DKey, Container, Containers, \ - VosValue, Overhead, ValType, VosValueError +import pytest +import yaml from storage_estimator.explorer import FileSystemExplorer -from storage_estimator.util import ObjectClass from storage_estimator.parse_csv import ProcessCSV +from storage_estimator.util import ObjectClass +from storage_estimator.vos_structures import (AKey, Container, Containers, DKey, Overhead, ValType, + VosObject, VosValue, VosValueError) + from .util import FileGenerator diff --git a/src/vos/storage_estimator/common/tests/util.py b/src/vos/storage_estimator/common/tests/util.py index 92006f268bb..79eaea7a9bb 100644 --- a/src/vos/storage_estimator/common/tests/util.py +++ b/src/vos/storage_estimator/common/tests/util.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' import os -import tempfile import shutil +import tempfile class FileGenerator(): diff --git a/src/vos/storage_estimator/common/util.py b/src/vos/storage_estimator/common/util.py index 6462c019181..b571d984a57 100644 --- a/src/vos/storage_estimator/common/util.py +++ b/src/vos/storage_estimator/common/util.py @@ -6,8 +6,8 @@ import os -import yaml +import yaml from storage_estimator.dfs_sb import VOS_SIZE, get_dfs_sb_obj from storage_estimator.vos_size import MetaOverhead from storage_estimator.vos_structures import Containers diff --git a/src/vos/storage_estimator/common/vos_size.py b/src/vos/storage_estimator/common/vos_size.py index 0b8cf9a9650..872fc76207a 100644 --- a/src/vos/storage_estimator/common/vos_size.py +++ b/src/vos/storage_estimator/common/vos_size.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import random import math +import random def convert(stat): diff --git a/src/vos/storage_estimator/daos_storage_estimator.py b/src/vos/storage_estimator/daos_storage_estimator.py index 2b9da5c4878..6e68df36f63 100755 --- a/src/vos/storage_estimator/daos_storage_estimator.py +++ b/src/vos/storage_estimator/daos_storage_estimator.py @@ -7,9 +7,9 @@ import argparse import sys -from storage_estimator.dfs_sb import get_dfs_example, print_daos_version, get_dfs_inode_akey -from storage_estimator.parse_csv import ProcessCSV +from storage_estimator.dfs_sb import get_dfs_example, get_dfs_inode_akey, print_daos_version from storage_estimator.explorer import FileSystemExplorer +from storage_estimator.parse_csv import ProcessCSV from storage_estimator.util import Common, ProcessBase tool_description = '''DAOS estimation tool diff --git a/src/vos/tests/bio_ut.c b/src/vos/tests/bio_ut.c index 2f89d67ffd8..52c834f50a3 100644 --- a/src/vos/tests/bio_ut.c +++ b/src/vos/tests/bio_ut.c @@ -34,7 +34,9 @@ ut_init(struct bio_ut_args *args) { int rc; - daos_debug_init(DAOS_LOG_DEFAULT); + rc = daos_debug_init(DAOS_LOG_DEFAULT); + if (rc) + return rc; rc = vos_self_init(db_path, false, BIO_STANDALONE_TGT_ID); if (rc) diff --git a/src/vos/tests/evt_stress.py b/src/vos/tests/evt_stress.py index d172d0ada86..ed05f61cc3f 100755 --- a/src/vos/tests/evt_stress.py +++ b/src/vos/tests/evt_stress.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """Run evt_ctl with a specific pattern that causes a segfault with the default sort algorithm""" +import argparse +import json import os from os.path import join -import json -import argparse class EVTStress(): diff --git a/src/vos/tests/vts_ilog.c b/src/vos/tests/vts_ilog.c index 6d4b0a5c88c..1b85d2cb683 100644 --- a/src/vos/tests/vts_ilog.c +++ b/src/vos/tests/vts_ilog.c @@ -158,7 +158,7 @@ fake_tx_log_add(struct umem_instance *umm, umem_off_t offset, uint32_t *tx_id, uint32_t idx; int rc; - rc = lrua_allocx(array, &idx, epoch, &entry); + rc = lrua_allocx(array, &idx, epoch, &entry, NULL); assert_rc_equal(rc, 0); assert_non_null(entry); diff --git a/src/vos/vos_dtx.c b/src/vos/vos_dtx.c index ccebf569082..3a0ad9cc938 100644 --- a/src/vos/vos_dtx.c +++ b/src/vos/vos_dtx.c @@ -40,8 +40,8 @@ enum { D_ASSERT(dae->dae_dth->dth_ent == dae); \ dae->dae_dth->dth_ent = NULL; \ } \ - D_DEBUG(DB_TRACE, "Evicting lid "DF_DTI": lid=%d\n", \ - DP_DTI(&DAE_XID(dae)), DAE_LID(dae)); \ + D_DEBUG(DB_TRACE, "Evicting lid "DF_DTI": lid=%lx\n", \ + DP_DTI(&DAE_XID(dae)), DAE_LID(dae) & DTX_LID_SOLO_MASK); \ d_list_del_init(&dae->dae_link); \ lrua_evictx(cont->vc_dtx_array, \ (DAE_LID(dae) & DTX_LID_SOLO_MASK) - DTX_LID_RESERVED, \ @@ -963,7 +963,7 @@ vos_dtx_alloc(struct vos_dtx_blob_df *dbd, struct dtx_handle *dth) cont = vos_hdl2cont(dth->dth_coh); D_ASSERT(cont != NULL); - rc = lrua_allocx(cont->vc_dtx_array, &idx, dth->dth_epoch, &dae); + rc = lrua_allocx(cont->vc_dtx_array, &idx, dth->dth_epoch, &dae, &dth->dth_local_stub); if (rc != 0) { /* The array is full, need to commit some transactions first */ if (rc == -DER_BUSY) @@ -1007,8 +1007,8 @@ vos_dtx_alloc(struct vos_dtx_blob_df *dbd, struct dtx_handle *dth) dae->dae_dbd = dbd; dae->dae_dth = dth; - D_DEBUG(DB_IO, "Allocated new lid DTX: "DF_DTI" lid=%d dae=%p" - " dae_dbd=%p\n", DP_DTI(&dth->dth_xid), DAE_LID(dae), dae, dbd); + D_DEBUG(DB_IO, "Allocated new lid DTX: "DF_DTI" lid=%lx, dae=%p, dae_dbd=%p\n", + DP_DTI(&dth->dth_xid), DAE_LID(dae) & DTX_LID_SOLO_MASK, dae, dbd); d_iov_set(&kiov, &DAE_XID(dae), sizeof(DAE_XID(dae))); d_iov_set(&riov, dae, sizeof(*dae)); @@ -1155,6 +1155,10 @@ vos_dtx_check_availability(daos_handle_t coh, uint32_t entry, found = lrua_lookupx(cont->vc_dtx_array, (entry & DTX_LID_SOLO_MASK) - DTX_LID_RESERVED, epoch, &dae); if (!found) { + D_ASSERTF(!(entry & DTX_LID_SOLO_FLAG), + "non-committed solo entry %lu must be there, epoch "DF_X64", boundary " + DF_X64"\n", entry & DTX_LID_SOLO_MASK, epoch, cont->vc_solo_dtx_epoch); + D_DEBUG(DB_TRACE, "Entry %d "DF_U64" not in lru array, it must be committed\n", entry, epoch); @@ -3238,3 +3242,15 @@ vos_dtx_cache_reset(daos_handle_t coh, bool force) return 0; } + +void +vos_dtx_renew_epoch(struct dtx_handle *dth) +{ + struct vos_dtx_act_ent *dae = dth->dth_ent; + + if (dae != NULL) + DAE_EPOCH(dae) = dth->dth_epoch; + + if (dth->dth_local_stub != NULL) + lrua_refresh_key(dth->dth_local_stub, dth->dth_epoch); +} diff --git a/src/vos/vos_io.c b/src/vos/vos_io.c index 644ffa771dc..e4ba73826fc 100644 --- a/src/vos/vos_io.c +++ b/src/vos/vos_io.c @@ -1492,28 +1492,24 @@ vos_fetch_begin(daos_handle_t coh, daos_unit_oid_t oid, daos_epoch_t epoch, &rc, false)) { if (rc == 0) { if (ioc->ic_read_ts_only) - goto set_ioc; + goto out; + if (ioc->ic_obj != NULL && has_uncertainty(ioc, &ioc->ic_obj->obj_ilog_info)) goto fetch_dkey; for (i = 0; i < iod_nr; i++) iod_empty_sgl(ioc, i); - goto set_ioc; } goto out; } fetch_dkey: if (dkey == NULL || dkey->iov_len == 0) { - if (ioc->ic_read_ts_only) - goto set_ioc; - D_GOTO(out, rc = -DER_INVAL); + if (!ioc->ic_read_ts_only) + rc = -DER_INVAL; + } else { + rc = dkey_fetch(ioc, dkey); } - rc = dkey_fetch(ioc, dkey); - if (rc != 0) - goto out; -set_ioc: - *ioh = vos_ioc2ioh(ioc); out: vos_dth_set(NULL, ioc->ic_cont->vc_pool->vp_sysdb); @@ -1532,9 +1528,13 @@ vos_fetch_begin(daos_handle_t coh, daos_unit_oid_t oid, daos_epoch_t epoch, if (rc != 0) { daos_recx_ep_list_free(ioc->ic_recx_lists, ioc->ic_iod_nr); ioc->ic_recx_lists = NULL; - return vos_fetch_end(vos_ioc2ioh(ioc), NULL, rc); + rc = vos_fetch_end(vos_ioc2ioh(ioc), NULL, rc); + *ioh = DAOS_HDL_INVAL; + } else { + *ioh = vos_ioc2ioh(ioc); } - return 0; + + return rc; } static umem_off_t diff --git a/utils/ansible/ftest/library/daos_hugepages.py b/utils/ansible/ftest/library/daos_hugepages.py index 49110232453..35d192b3035 100644 --- a/utils/ansible/ftest/library/daos_hugepages.py +++ b/utils/ansible/ftest/library/daos_hugepages.py @@ -9,9 +9,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import datetime import os import re -import datetime import subprocess # nosec B404 from ansible.module_utils.basic import AnsibleModule diff --git a/utils/cq/d_logging_check.py b/utils/cq/d_logging_check.py new file mode 100755 index 00000000000..fca019815b8 --- /dev/null +++ b/utils/cq/d_logging_check.py @@ -0,0 +1,454 @@ +#!/usr/bin/env python + +"""Check and re-write DAOS logging macros to preferred coding style + +Apply some checks for the macros, partly for correctness but also for consistency. Some errors +can be fixed however not all can. Code formatting will be incorrect after fixing so clang-format +(or git-clang-format -f) should be used after this. + +Use --fix to apply any correctness fixes that can be applied. +Use --correct to apply any style changes that can be applied. Style changes will be applied +if fixes are being applied. +""" + +import argparse +import inspect +import io +import os +import re +import sys + +ARGS = None + + +class FileLine(): + """One line from a file""" + + def __init__(self, file_object, line, lineno): + self._fo = file_object + self._lineno = lineno + + # Text as it appears in the source + self._line = line + # Striped text + self._code = line.strip() + self.modified = False + self.corrected = False + + def startswith(self, string): + """Starts-with method""" + return self._code.startswith(string) + + def endswith(self, string): + """Ends-with method""" + return self._code.endswith(string) + + def count(self, string): + """Returns count of substring""" + return self._code.count(string) + + def __contains__(self, string): + return string in self._code + + def __str__(self): + if self.modified or (self.corrected and ARGS.correct): + return f'{self._code}\n' + return self._line + + def expand(self): + """Expand line to end""" + while not self._code.endswith(';'): + to_add = str(next(self._fo)) + self._code += to_add.strip() + self._line += to_add + + def write(self, flo): + """Write line to file""" + flo.write(str(self)) + + def correct(self, new_code): + """Apply corrections to a line. + + These will only be applied if fixes are also being applied. + """ + self._code = new_code + self.corrected = True + self.note('Optional fixes') + + def fix(self, new_code): + """Mark a line as updated""" + self.correct(new_code) + self.modified = True + self.note('Required fixes') + + def raw(self): + """Returns the code""" + return self._code + + def warning(self, msg): + """Show a warning""" + print(f'{self._fo.fname}:{self._lineno} {msg}') + if ARGS.github: + fn_name = inspect.stack()[1].function + fn_name = fn_name.replace('_', '-') + print(f'::warning file={self._fo.fname},line={self._lineno},::{fn_name}, {msg}') + + def note(self, msg): + """Show a note""" + print(f'{self._fo.fname}:{self._lineno} {msg}') + + +class FileParser: + """One source file""" + + def __init__(self, fname): + self.fname = fname + self.lines = [] + self._index = None + with open(fname, 'r') as fd: + lineno = 1 + for line in fd: + self.lines.append(FileLine(self, line, lineno)) + lineno += 1 + + def __iter__(self): + self._index = -1 + return self + + def __next__(self): + self._index += 1 + try: + return self.lines[self._index] + except IndexError as exc: + raise StopIteration from exc + + +# Logging macros where the new-line is added if missing. +PREFIXES = ['D_ERROR', 'D_WARN', 'D_INFO', 'D_NOTE', 'D_ALERT', 'D_CRIT', 'D_FATAT', 'D_EMIT', + 'D_TRACE_INFO', 'D_TRACE_NOTE', 'D_TRACE_WARN', 'D_TRACE_ERROR', 'D_TRACE_ALERT', + 'D_TRACE_CRIT', 'D_TRACE_FATAL', 'D_TRACE_EMIT', 'RPC_TRACE', 'RPC_ERROR', + 'VOS_TX_LOG_FAIL', 'VOS_TX_TRACE_FAIL', 'D_DEBUG', 'D_CDEBUG'] + +# Logging macros where a new-line is always added. +PREFIXES_NNL = ['DFUSE_LOG_WARNING', 'DFUSE_LOG_ERROR', 'DFUSE_LOG_DEBUG', 'DFUSE_LOG_INFO', + 'DFUSE_TRA_WARNING', 'DFUSE_TRA_ERROR', 'DFUSE_TRA_DEBUG', 'DFUSE_TRA_INFO', + 'DH_PERROR_SYS', 'DH_PERROR_DER', 'DL_CDEBUG'] + +for prefix in ['DL', 'DHL', 'DS', 'DHS']: + for suffix in ['ERROR', 'WARN', 'INFO']: + PREFIXES_NNL.append(f'{prefix}_{suffix}') + +PREFIXES_ALL = PREFIXES.copy() +PREFIXES_ALL.extend(PREFIXES_NNL) + + +class AllChecks(): + """All the checks in one class""" + + def __init__(self, file_object): + self._fo = file_object + self.line = '' + self._output = io.StringIO() + self.modified = False + self.corrected = False + + def run_all_checks(self): + """Run everything + + Iterate over the input file line by line checking for logging use and run checks on lines + where the macros are used. Ignore lines within macro definitions. + """ + prev_macro = False + for line in self._fo: + if line.endswith('\\'): + prev_macro = True + # line.note('Part of macro, not checking') + line.write(self._output) + continue + + if prev_macro: + prev_macro = False + line.write(self._output) + continue + + if not any(map(line.startswith, PREFIXES_ALL)): + line.write(self._output) + continue + + line.expand() + + self.check_print_string(line) + self.check_quote(line) + self.check_return(line) + self.check_df_rc_dot(line) + self.check_for_newline(line) + self.check_df_rc(line) + self.remove_trailing_period(line) + self.check_quote(line) + self.check_failed(line) + + line.write(self._output) + if line.modified: + self.modified = True + if line.corrected: + self.corrected = True + + def save(self, fname): + """Save new file to file""" + if not self.modified and not self.corrected: + return + with open(fname, 'w') as fd: + fd.write(self._output.getvalue()) + print('Changes saved, run git-clang-format to format them.') + + def check_print_string(self, line): + """Check for %s in message""" + if line.startswith('DH_PERROR'): + return + count = line.count('%s') + if count == 0: + return + if count == 1 and line.count('strerror') == 1: + return + if '?' in line: + line.note('Message uses %s with trigraph') + else: + line.note('Message uses %s without trigraph') + + def check_quote(self, line): + """Check for double quotes in message""" + if '""' not in line: + return + + # Do not remove if used in tri-graphs. + if ': ""' in line or '? ""' in line or '\\""' in line: + return + + line.correct(line.raw().replace('""', '')) + + def check_return(self, line): + """Check for one return character""" + max_newlines = 1 + code = line.raw() + if any(map(code.startswith, PREFIXES_NNL)): + max_newlines = 0 + + if '"%s",' in code: + line.note('Use of %s at end of log-line, unable to check') + return + + count = code.count('\\n') + # Some logging calls contain multiple new-line characters and that's OK, as long as one of + # them isn't at the end of a line. + if max_newlines == 0 and count > max_newlines: + line.warning("Line contains too many newlines") + + def check_df_rc_dot(self, line): + """Check there is no . after DF_RC""" + code = line.raw() + count = code.count('DF_RC') + if count == 0: + return + code = code.replace('DF_RC ', 'DF_RC') + if 'DF_RC".\\n"' not in code: + return + code = code.replace('DF_RC".\\n"', 'DF_RC"\\n"') + line.warning('Extra . after DP_RC macro') + line.fix(code) + + def check_df_rc(self, line): + r"""Check for text before DF_RC macro + + Re-flow lines that use DF_RC so that they are of the form '...: " DF_RC,' + There should be a ": " before the DF_RC. + There should not be a space before the : + There should not be other special characters used. + The variable name should not be printed + + """ + code = line.raw() + count = code.count('DF_RC') + if count == 0: + return + if count != 1: + line.note('Cannot check lines with multiple DF_RC') + return + if not code.endswith('));'): + line.note('Unable to check DF_RC') + return + + # Remove any spaces around macros as these may or may not be present. This updated input + # is used for the update check at the end so white-space differences here will not cause + # code to be re-written. + code = re.sub(r' ?DF_RC ?', 'DF_RC', code) + code = re.sub(r' ?DP_RC ?', 'DP_RC', code) + + # Check that DF_RC is at the end of the line, it should be. + + if '\\n' in code: + if 'DF_RC"\\n"' not in code: + line.note('DF_RC is not at end of line') + return + else: + if 'DF_RC,' not in code: + line.note('DF_RC is not at end of line') + return + + # Extract the variable name + parts = code[:-3].split('(') + if not parts[-2].endswith('DP_RC'): + line.note('Function in DF_RC call') + return + var_name = parts.pop() + new_code = '('.join(parts) + assert new_code.endswith('DP_RC') + new_code = new_code[:-5] + + # Strip out the string formatting message + parts = code.split('DF_RC') + assert len(parts) == 2 + msg = parts[0] + + assert msg.endswith('"') + msg = msg[:-1] + + # Check what comes before DF_RC in the message, and strip any trailing punctuation. + imsg = None + while imsg != msg: + imsg = msg + if any(map(msg.endswith, [' ', '=', '.', ',', ':', ';'])): + msg = msg[:-1] + if msg.endswith(var_name): + msg = msg[:-len(var_name)] + if msg.endswith('rc'): + msg = msg[:-2] + + # Put it all back together with consistent style. + new_code = f'{msg}: "DF_RC{parts[1]}' + + if any(map(line.startswith, ['D_ERROR', 'D_WARN', 'D_INFO'])): + new_code = f'{msg}"DF_RC{parts[1]}' + if line.startswith('D_ERROR'): + new_code = f'DL_ERROR({var_name}, {new_code[8:]}' + elif line.startswith('D_WARN'): + new_code = f'DL_WARN({var_name}, {new_code[7:]}' + else: + new_code = f'DL_INFO({var_name}, {new_code[7:]}' + new_code = new_code.replace('DF_RC', '') + new_code = new_code.replace(f',DP_RC({var_name})', '') + + if new_code != code: + line.correct(new_code) + + def check_for_newline(self, line): + """Remove optional new-lines""" + code = line.raw() + + if any(map(code.startswith, PREFIXES_NNL)): + return + + if '\\"\\n' in code: + line.note('Unable to check for newlines') + return + + new_code = code.replace('"\\n",', ',') + new_code = new_code.replace('\\n",', '",') + new_code = new_code.replace('"\\n")', ')') + new_code = new_code.replace('\\n")', '")') + if new_code != code: + line.correct(new_code) + + def remove_trailing_period(self, line): + """Remove . from the end of a line""" + code = line.raw() + + new_code = code + before_code = None + while new_code != before_code: + before_code = new_code + new_code = re.sub(r'\.",', '",', new_code) + new_code = re.sub(r'\."\)', '")', new_code) + + if new_code != code: + line.correct(new_code) + + def check_failed(self, line): + """Check for 'Failed' with uppercase F + + Lots of message are of the form 'function() failed' but some use Failed. + """ + code = line.raw() + + if 'Failed' not in code: + return + if '"Failed' in code: + return + if 'Failed to' in code: + return + + print(code) + line.note('Failed') + + +def one_entry(fname): + """Process one path entry + + Returns true if there are un-fixed errors. + """ + if not any(map(fname.endswith, ['.c', '.h'])): + return False + + if any(map(fname.endswith, ['pb-c.c', 'pb-c..h'])): + return False + + if ARGS.verbose: + print(f'Checking {fname}') + filep = FileParser(fname) + + checks = AllChecks(filep) + + checks.run_all_checks() + + if (ARGS.fix and checks.modified) or (ARGS.correct and checks.corrected): + print(f'Saving updates to {fname}') + checks.save(fname) + return False + + if checks.modified and not ARGS.fix: + return True + return False + + +def main(): + """Do something""" + parser = argparse.ArgumentParser(description='Verify DAOS logging in source tree') + parser.add_argument('--fix', action='store_true', help='Apply required fixes') + parser.add_argument('--correct', action='store_true', help='Apply optional fixes') + parser.add_argument('--github', action='store_true') + parser.add_argument('-v', '--verbose', action='store_true') + parser.add_argument('files', nargs='*') + + global ARGS + + ARGS = parser.parse_args() + unfixed_errors = False + + for fname in ARGS.files: + if os.path.isfile(fname): + one_entry(fname) + else: + for root, dirs, files in os.walk(fname): + for name in files: + if one_entry(os.path.join(root, name)): + unfixed_errors = True + if '.git' in dirs: + dirs.remove('.git') + if root == 'src/control' and 'vendor' in dirs: + dirs.remove('vendor') + if unfixed_errors: + print('Required fixes not applied') + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/utils/cq/daos_pylint.py b/utils/cq/daos_pylint.py index 05393e22df0..22e1fbc997c 100755 --- a/utils/cq/daos_pylint.py +++ b/utils/cq/daos_pylint.py @@ -1,21 +1,22 @@ #!/usr/bin/env python3 """Wrapper script for calling pylint""" +import argparse +import json import os -import sys import re -from collections import Counter -import tempfile import subprocess # nosec -import argparse -import json +import sys +import tempfile +from collections import Counter + for arg in sys.argv: if arg.startswith('--import='): sys.path.append(arg[9:]) try: + from pylint.constants import full_version from pylint.lint import Run from pylint.reporters.collecting_reporter import CollectingReporter - from pylint.constants import full_version except ImportError: if os.path.exists('venv'): @@ -23,9 +24,9 @@ f'python{sys.version_info.major}.{sys.version_info.minor}', 'site-packages')) try: + from pylint.constants import full_version from pylint.lint import Run from pylint.reporters.collecting_reporter import CollectingReporter - from pylint.constants import full_version except ImportError: print('detected venv unusable, install pylint to enable this check') sys.exit(0) diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index 071191dece9..a5d00a92fa7 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -1,4 +1,4 @@ -# Packages needed to pylint code. Most of these are needed for test rather than build. +# Packages needed to check code. These are needed for linting rather than build. # 94.0 is reporting a couple of errors so stick on 93. avocado-framework<94 avocado-framework-plugin-result-html<94 @@ -6,4 +6,10 @@ avocado-framework-plugin-varianter-yaml-to-mux<94 clustershell paramiko pyenchant -pylint +## flake8 6 removed --diff option which breaks flake precommit hook. +## https://github.com/pycqa/flake8/issues/1389 https://github.com/PyCQA/flake8/pull/1720 +flake8<6.0.0 +isort==5.12 +pylint==3.0.2 +yamllint==1.32.0 +codespell==2.2.6 diff --git a/utils/githooks/pre-commit.d/20-flake.sh b/utils/githooks/pre-commit.d/20-flake.sh index f9973bdabc4..8df37d63384 100755 --- a/utils/githooks/pre-commit.d/20-flake.sh +++ b/utils/githooks/pre-commit.d/20-flake.sh @@ -18,6 +18,7 @@ set -ue echo "Flake8:" if ! command -v flake8 > /dev/null 2>&1; then echo " No flake checking, install flake8 command to improve pre-commit checks" + echo "python3 -m pip install -r ./utils/cq/requirements.txt" exit 0 fi diff --git a/utils/githooks/pre-commit.d/20-isort.sh b/utils/githooks/pre-commit.d/20-isort.sh new file mode 100755 index 00000000000..8b2e496badc --- /dev/null +++ b/utils/githooks/pre-commit.d/20-isort.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# Runs isort for the DAOS project. + +set -ue + +if ! command -v isort > /dev/null 2>&1 +then + echo "No isort checking, install isort command to improve pre-commit checks" + echo "python3 -m pip install -r ./utils/cq/requirements.txt" + exit 0 +fi + +echo "isort:" + +echo " Running isort for python imports." +isort --jobs 8 --check-only . diff --git a/utils/githooks/pre-commit.d/30-yamllint.sh b/utils/githooks/pre-commit.d/30-yamllint.sh index 3b5efdb7e93..d395dc25564 100755 --- a/utils/githooks/pre-commit.d/30-yamllint.sh +++ b/utils/githooks/pre-commit.d/30-yamllint.sh @@ -10,7 +10,7 @@ set -ue if ! command -v yamllint > /dev/null 2>&1 then echo "No yaml checking, install yamllint command to improve pre-commit checks" - echo "python3 -m pip install yamllint" + echo "python3 -m pip install -r ./utils/cq/requirements.txt" exit 0 fi diff --git a/utils/node_local_test.py b/utils/node_local_test.py index b05f0110929..1b80f1ae0a0 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -10,29 +10,30 @@ # pylint: disable=too-many-lines -import os -from os.path import join -import sys -import time -import uuid -import json +import argparse import copy -import signal -import pprint -import stat import errno -import argparse -import threading import functools -import traceback -import subprocess # nosec -import tempfile +import json +import os import pickle # nosec +import pprint import re import shutil -import xattr +import signal +import stat +import subprocess # nosec +import sys +import tempfile +import threading +import time +import traceback +import uuid +from os.path import join + import junit_xml import tabulate +import xattr import yaml diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 960ab7868ea..3fc06773726 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -14,8 +14,8 @@ %endif Name: daos -Version: 2.4.0 -Release: 4%{?relval}%{?dist} +Version: 2.4.1 +Release: 1%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -72,7 +72,7 @@ BuildRequires: libisa-l_crypto-devel BuildRequires: libisal-devel BuildRequires: libisal_crypto-devel %endif -BuildRequires: daos-raft-devel = 0.10.1-1.408.g9524cdb%{?dist} +BuildRequires: daos-raft-devel = 0.10.1-2.411.gefa15f4%{?dist} BuildRequires: openssl-devel BuildRequires: libevent-devel BuildRequires: libyaml-devel @@ -555,6 +555,13 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Fri Dec 08 2023 Phillip Henderson 2.4.1-1 +- First release candidate for 2.4.1 + +* Wed Dec 06 2023 Brian J. Murrell 2.4.0-3 +- Update for EL 8.8 and Leap 15.5 +- Update raft to 0.10.1-2.411.gefa15f4 + * Tue Nov 28 2023 Jerome Soumagne 2.4.0-4 - Bump mercury min version to 2.3.1 diff --git a/utils/rpms/packaging/Dockerfile.centos.7 b/utils/rpms/packaging/Dockerfile.centos.7 index cdfb7f69784..189ea1e4c61 100644 --- a/utils/rpms/packaging/Dockerfile.centos.7 +++ b/utils/rpms/packaging/Dockerfile.centos.7 @@ -5,9 +5,31 @@ # # Pull base image -FROM centos:7 +FROM centos:centos7 LABEL maintainer="daos@daos.groups.io" +# Use local repo server if present +ARG REPO_FILE_URL +RUN set -e; \ + if [ -n "$REPO_FILE_URL" ]; then \ + cd /etc/yum.repos.d/ && \ + curl -k -f -o daos_ci-centos7-artifactory.repo.tmp \ + "$REPO_FILE_URL"daos_ci-centos7-artifactory.repo && \ + for file in *.repo; do \ + true > $file; \ + done; \ + mv daos_ci-centos7-artifactory.repo{.tmp,}; \ + fi; \ + yum -y install dnf; \ + yum clean all; \ + dnf --disablerepo \*epel\* -y install epel-release \ + dnf-plugins-core; \ + if [ -n "$REPO_FILE_URL" ]; then \ + dnf -y --quiet config-manager --disable epel; \ + fi; \ + dnf -y update epel-release; \ + dnf -y clean all + # use same UID as host and default value of 1000 if not specified ARG UID=1000 @@ -15,9 +37,9 @@ ARG UID=1000 #Nothing to do for CentOS # Install basic tools -RUN yum install -y epel-release -RUN yum install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ - git python-srpm-macros dnf +RUN dnf install -y epel-release +RUN dnf install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ + git python-srpm-macros dnf && dnf -y clean all # Add build user (to keep rpmbuild happy) ENV USER build diff --git a/utils/rpms/packaging/Dockerfile.mockbuild b/utils/rpms/packaging/Dockerfile.mockbuild index 533c7c5fda5..5c6475db1f5 100644 --- a/utils/rpms/packaging/Dockerfile.mockbuild +++ b/utils/rpms/packaging/Dockerfile.mockbuild @@ -5,7 +5,7 @@ # # Pull base image -ARG FVERSION=38 # this should be changed to 'latest' and some point soon +ARG FVERSION=latest FROM fedora:$FVERSION # Needed for later use of FVERSION ARG FVERSION @@ -15,16 +15,19 @@ LABEL maintainer="daos@daos.groups.io" ARG REPO_FILE_URL RUN if [ -n "$REPO_FILE_URL" ]; then \ cd /etc/yum.repos.d/ && \ - curl -f -o daos_ci-fedora-artifactory.repo.tmp \ + curl -k -f -o daos_ci-fedora-artifactory.repo.tmp \ "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \ - rm -f *.repo && \ + for file in *.repo; do \ + true > $file; \ + done; \ mv daos_ci-fedora-artifactory.repo{.tmp,}; \ fi # Install basic tools RUN dnf -y install mock make \ rpm-build createrepo rpmlint redhat-lsb-core git \ - python-srpm-macros rpmdevtools + python-srpm-macros rpmdevtools && \ + dnf -y clean all # use same UID as host and default value of 1000 if not specified ARG UID=1000 @@ -45,7 +48,8 @@ RUN dnf -y upgrade && \ # https://github.com/rpm-software-management/rpmlint/pull/795 in it # But make sure to patch after dnf upgrade so that an upgraded rpmlint # RPM doesn't wipe out our patch -COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch . +ARG PACKAGINGDIR=packaging +COPY ${PACKAGINGDIR}/rpmlint--ignore-unused-rpmlintrc.patch . RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \ if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then \ if ! patch -p1; then \ diff --git a/utils/rpms/packaging/Makefile_distro_vars.mk b/utils/rpms/packaging/Makefile_distro_vars.mk index e9e0784668a..85cd97dab71 100644 --- a/utils/rpms/packaging/Makefile_distro_vars.mk +++ b/utils/rpms/packaging/Makefile_distro_vars.mk @@ -71,6 +71,14 @@ DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 15.4 SED_EXPR := 1p endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.5-x86_64) +VERSION_ID := 15.5 +DISTRO_ID := sl15.5 +DISTRO_BASE := LEAP_15 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 15.5 +SED_EXPR := 1p +endif endif ifeq ($(ID),centos) ID = el diff --git a/utils/rpms/packaging/Makefile_packaging.mk b/utils/rpms/packaging/Makefile_packaging.mk index 7adda97cd9c..d045f42c0e4 100644 --- a/utils/rpms/packaging/Makefile_packaging.mk +++ b/utils/rpms/packaging/Makefile_packaging.mk @@ -53,6 +53,7 @@ RPM_BUILD_OPTIONS := $(BUILD_DEFINES) GIT_DIFF_EXCLUDES := $(PATCH_EXCLUDE_FILES:%=':!%') endif +FVERSION ?= latest COMMON_RPM_ARGS := --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES) SPEC := $(shell if [ -f $(NAME)-$(DISTRO_BASE).spec ]; then echo $(NAME)-$(DISTRO_BASE).spec; else echo $(NAME).spec; fi) VERSION = $(eval VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p'))$(VERSION) @@ -64,10 +65,16 @@ RPMS = $(eval RPMS := $(addsuffix .rpm,$(addprefix _topdir/RPMS/x86 DEB_TOP := _topdir/BUILD DEB_BUILD := $(DEB_TOP)/$(NAME)-$(VERSION) DEB_TARBASE := $(DEB_TOP)/$(DEB_NAME)_$(VERSION) -SOURCE ?= $(eval SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\#/\\\#/g' -e 's/.*: *//'))$(SOURCE) -PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) -OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/d' -e p))$(OTHER_SOURCES) -SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES) $(OTHER_SOURCES)) +REAL_SOURCE ?= $(eval REAL_SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\#/\\\#/g' -e 's/Source.*: *//'))$(REAL_SOURCE) +ifeq ($(ID_LIKE),debian) +ifneq ($(DEB_SOURCE),) +SOURCE ?= $(DEB_SOURCE) +endif +endif +SOURCE ?= $(REAL_SOURCE) +PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) +OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e '/^Patch.*:/d' -e 's/Source.*: *//' -e 's/.*\///' -e p))$(OTHER_SOURCES) +SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE) $(OTHER_SOURCES)) $(PATCHES)) ifeq ($(ID_LIKE),debian) DEBS := $(addsuffix _$(VERSION)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' $(TOPDIR)/debian/control)) DEB_PREV_RELEASE := $(shell cd $(TOPDIR) && dpkg-parsechangelog -S version) @@ -154,9 +161,9 @@ ifeq ($(DL_NAME),) DL_NAME = $(NAME) endif -$(notdir $(SOURCE)): $(SPEC) $(CALLING_MAKEFILE) +$(notdir $(SOURCE) $(OTHER_SOURCES) $(REAL_SOURCE)): $(SPEC) $(CALLING_MAKEFILE) # TODO: need to clean up old ones - $(SPECTOOL) -g $(SPEC) + $(SPECTOOL) $(COMMON_RPM_ARGS) -g $(SPEC) $(DEB_TOP)/%: % | $(DEB_TOP)/ @@ -181,27 +188,29 @@ $(DEB_TOP)/.patched: $(PATCHES) check-env deb_detar | \ $(DEB_BUILD)/debian/ mkdir -p ${DEB_BUILD}/debian/patches mkdir -p $(DEB_TOP)/patches - for f in $(PATCHES); do \ - rm -f $(DEB_TOP)/patches/*; \ - if git mailsplit -o$(DEB_TOP)/patches < "$$f" ;then \ - fn=$$(basename "$$f"); \ - for f1 in $(DEB_TOP)/patches/*;do \ - [ -e "$$f1" ] || continue; \ - f1n=$$(basename "$$f1"); \ - echo "$${fn}_$${f1n}" >> $(DEB_BUILD)/debian/patches/series ; \ - mv "$$f1" $(DEB_BUILD)/debian/patches/$${fn}_$${f1n}; \ - done; \ - else \ - fb=$$(basename "$$f"); \ - cp "$$f" $(DEB_BUILD)/debian/patches/ ; \ - echo "$$fb" >> $(DEB_BUILD)/debian/patches/series ; \ - if ! grep -q "^Description:\|^Subject:" "$$f" ;then \ - sed -i '1 iSubject: Auto added patch' \ - "$(DEB_BUILD)/debian/patches/$$fb" ;fi ; \ - if ! grep -q "^Origin:\|^Author:\|^From:" "$$f" ;then \ - sed -i '1 iOrigin: other' \ - "$(DEB_BUILD)/debian/patches/$$fb" ;fi ; \ - fi ; \ + for f in $(PATCHES); do \ + rm -f $(DEB_TOP)/patches/*; \ + if git mailsplit -o$(DEB_TOP)/patches < "$$f"; then \ + fn=$$(basename "$$f"); \ + for f1 in $(DEB_TOP)/patches/*;do \ + [ -e "$$f1" ] || continue; \ + f1n=$$(basename "$$f1"); \ + echo "$${fn}_$${f1n}" >> $(DEB_BUILD)/debian/patches/series; \ + mv "$$f1" $(DEB_BUILD)/debian/patches/$${fn}_$${f1n}; \ + done; \ + else \ + fb=$$(basename "$$f"); \ + cp "$$f" $(DEB_BUILD)/debian/patches/; \ + echo "$$fb" >> $(DEB_BUILD)/debian/patches/series; \ + if ! grep -q "^Description:\|^Subject:" "$$f"; then \ + sed -i '1 iSubject: Auto added patch' \ + "$(DEB_BUILD)/debian/patches/$$fb"; \ + fi; \ + if ! grep -q "^Origin:\|^Author:\|^From:" "$$f"; then \ + sed -i '1 iOrigin: other' \ + "$(DEB_BUILD)/debian/patches/$$fb"; \ + fi; \ + fi; \ done touch $@ @@ -360,6 +369,7 @@ endif podman_chrootbuild: if ! podman build --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \ + --build-arg FVERSION=$(FVERSION) \ -t $(subst +,-,$(CHROOT_NAME))-chrootbuild \ -f packaging/Dockerfile.mockbuild .; then \ echo "Container build failed"; \ @@ -377,7 +387,9 @@ podman_chrootbuild: exit 1; \ fi; \ rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm | \ - grep -v -e debuginfo -e debugsource -e src.rpm)' + grep -v -e debuginfo -e debugsource -e src.rpm)'; then \ + exit 1; \ + fi docker_chrootbuild: if ! $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ @@ -466,6 +478,9 @@ show_rpms: show_source: @echo '$(SOURCE)' +show_real_source: + @echo '$(REAL_SOURCE)' + show_patches: @echo '$(PATCHES)' diff --git a/utils/rpms/packaging/rpm_chrootbuild b/utils/rpms/packaging/rpm_chrootbuild index 5e79337cae0..9661047727b 100755 --- a/utils/rpms/packaging/rpm_chrootbuild +++ b/utils/rpms/packaging/rpm_chrootbuild @@ -2,17 +2,10 @@ set -uex -mock_config_dir="${WORKSPACE:-$PWD}/mock" -original_cfg_file="/etc/mock/${CHROOT_NAME}.cfg" -cfg_file="$mock_config_dir/${CHROOT_NAME}.cfg" -mkdir -p "$mock_config_dir" -ln -sf /etc/mock/templates "$mock_config_dir/" -ln -sf /etc/mock/logging.ini "$mock_config_dir/" - -cp "$original_cfg_file" "$cfg_file" +cp /etc/mock/"$CHROOT_NAME".cfg mock.cfg if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then - cat <> "$cfg_file" + cat <> mock.cfg config_opts['module_setup_commands'] = [ ('enable', 'javapackages-tools:201801'), ('disable', 'go-toolset') @@ -25,12 +18,8 @@ if [[ $CHROOT_NAME == *epel-7-x86_64 ]]; then MOCK_OPTIONS="--dnf --no-bootstrap-chroot${MOCK_OPTIONS:+ }$MOCK_OPTIONS" fi -# disable bootstrap_image -# https://github.com/rpm-software-management/mock/issues/1184 -MOCK_OPTIONS="--no-bootstrap-image${MOCK_OPTIONS:+ }$MOCK_OPTIONS" - # Allow BR: foo-devel < 1.2 to work when foo-devel-1.3 is actually available -cat <> "$cfg_file" +cat <> mock.cfg config_opts['dnf.conf'] += """ [main] best=0 @@ -41,7 +30,7 @@ EOF repo_adds=() repo_dels=() -echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file" +echo -e "config_opts['yum.conf'] += \"\"\"\n" >> mock.cfg if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then repo_dels+=("--disablerepo=\*") @@ -64,7 +53,7 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then REPO_FILE_URL="file://$(readlink -e "$REPO_FILES_PR")/" fi fi - curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-mock-artifactory.repo >> "$cfg_file" + curl -sSf "$REPO_FILE_URL"daos_ci-"${CHROOT_NAME%-*}".repo >> mock.cfg repo_adds+=("--enablerepo *-artifactory") fi fi @@ -85,7 +74,7 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do name=$repo:$branch:$build_number\n\ baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/\n\ enabled=1\n\ -gpgcheck=False\n" >> "$cfg_file" +gpgcheck=False\n" >> mock.cfg done for repo in $JOB_REPOS; do repo_name=${repo##*://} @@ -93,19 +82,35 @@ for repo in $JOB_REPOS; do repo_adds+=("--enablerepo $repo_name") echo -e "[${repo_name//@/_}]\n\ name=${repo_name}\n\ -baseurl=${repo}\n\ -enabled=1\n" >> "$cfg_file" +baseurl=${repo//\//%2F}\n\ +enabled=1\n" >> mock.cfg done -echo "\"\"\"" >> "$cfg_file" +echo "\"\"\"" >> mock.cfg if [ -n "$DISTRO_VERSION" ]; then releasever_opt=("--config-opts=releasever=$DISTRO_VERSION") fi -# shellcheck disable=SC2086 -if ! eval mock --configdir "$mock_config_dir" -r "${CHROOT_NAME}" \ - ${repo_dels[*]} ${repo_adds[*]} --disablerepo=\*-debug* \ - "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET"; then - # Debug information for filing bugs on mock - rpm -q mock - mock --debug-config + +bs_dir=/scratch/mock/cache/"${CHROOT_NAME}"-bootstrap +if ls -l /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz; then + mkdir -p "/var/cache/mock/${CHROOT_NAME}-bootstrap" + flock "$bs_dir" -c "cp -a $bs_dir/root_cache /var/cache/mock/${CHROOT_NAME}-bootstrap" +fi + +rc=0 +# shellcheck disable=SC2086,SC2048 +if ! eval mock -r mock.cfg ${repo_dels[*]} ${repo_adds[*]} --no-clean \ + --disablerepo=\*-debug* ${releasever_opt[*]} $MOCK_OPTIONS \ + $RPM_BUILD_OPTIONS "$TARGET"; then + rc=${PIPESTATUS[0]} fi + +if ls -l /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz && + [ -d /scratch/ ]; then + mkdir -p /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/ + if ! cmp /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz "$bs_dir"/root_cache/cache.tar.gz; then + flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/" + fi +fi + +exit "$rc" diff --git a/utils/run_utest.py b/utils/run_utest.py index 6c2068b0698..4b0d7f941e1 100755 --- a/utils/run_utest.py +++ b/utils/run_utest.py @@ -6,18 +6,19 @@ Test script for running all DAOS unit tests """ +import argparse +import json # pylint: disable=broad-except import os -import sys -import json -import argparse -import shutil import re +import shutil import subprocess # nosec +import sys import tempfile import traceback -from junit_xml import TestSuite, TestCase + import yaml +from junit_xml import TestCase, TestSuite def check_version(): diff --git a/utils/scripts/helpers/repo-helper-el8.sh b/utils/scripts/helpers/repo-helper-el8.sh index f1a3459d64b..2b512e3189f 100755 --- a/utils/scripts/helpers/repo-helper-el8.sh +++ b/utils/scripts/helpers/repo-helper-el8.sh @@ -97,8 +97,9 @@ for repo in $REPOS; do name=$repo:$branch:$build_number\n\ baseurl=${JENKINS_URL}$daos_base$repo/job/$branch/$build_number$artifacts\n\ enabled=1\n\ -gpgcheck=False\n" >> /etc/yum.repos.d/$repo:$branch:$build_number.repo - cat /etc/yum.repos.d/$repo:$branch:$build_number.repo +gpgcheck=False\n +module_hotfixes=true\n" >> /etc/yum.repos.d/"$repo:$branch:$build_number".repo + cat /etc/yum.repos.d/"$repo:$branch:$build_number".repo save_repos+=("$repo:$branch:$build_number") done diff --git a/utils/scripts/helpers/repo-helper-el9.sh b/utils/scripts/helpers/repo-helper-el9.sh index 60e1314bea2..fe1959589ef 100644 --- a/utils/scripts/helpers/repo-helper-el9.sh +++ b/utils/scripts/helpers/repo-helper-el9.sh @@ -75,7 +75,7 @@ if [ ! -f /etc/fedora-release ]; then PT_REPO=crb fi dnf install epel-release - dnf config-manager --enable $PT_REPO + dnf config-manager --enable "$PT_REPO" fi dnf clean all @@ -83,6 +83,10 @@ daos_base="job/daos-stack/job/" artifacts="/artifact/artifacts/el$MAJOR_VER/" save_repos=() for repo in $REPOS; do + # don't install daos@ repos since we are building daos + if [[ $repo = daos@* ]]; then + continue + fi branch="master" build_number="lastSuccessfulBuild" if [[ $repo = *@* ]]; then @@ -97,8 +101,9 @@ for repo in $REPOS; do name=$repo:$branch:$build_number\n\ baseurl=${JENKINS_URL}$daos_base$repo/job/$branch/$build_number$artifacts\n\ enabled=1\n\ -gpgcheck=False\n" >> /etc/yum.repos.d/$repo:$branch:$build_number.repo - cat /etc/yum.repos.d/$repo:$branch:$build_number.repo +gpgcheck=False\n +module_hotfixes=true\n" >> /etc/yum.repos.d/"$repo:$branch:$build_number".repo + cat /etc/yum.repos.d/"$repo:$branch:$build_number".repo save_repos+=("$repo:$branch:$build_number") done diff --git a/utils/scripts/helpers/repo-helper-leap15.sh b/utils/scripts/helpers/repo-helper-leap15.sh index 4d46d8db540..6afb27d0c3c 100755 --- a/utils/scripts/helpers/repo-helper-leap15.sh +++ b/utils/scripts/helpers/repo-helper-leap15.sh @@ -7,7 +7,7 @@ set -uex # for custom packages if present. : "${REPO_FILE_URL:=}" -: "${BASE_DISTRO:=opensuse/leap:15.3}" +: "${BASE_DISTRO:=opensuse/leap:15.5}" : "${JENKINS_URL:=}" : "${REPOS:=}" @@ -135,12 +135,13 @@ for repo in $REPOS; do name=$repo:$branch:$build_number\n\ baseurl=${JENKINS_URL}$daos_base$repo/job/$branch/$build_number$artifacts\n\ enabled=1\n\ -gpgcheck=False\n" >> $repos_dir$repo:$branch:$build_number.repo - cat $repos_dir$repo:$branch:$build_number.repo +gpgcheck=False\n +module_hotfixes=true\n" >> "$repos_dir$repo:$branch:$build_number".repo + cat "$repos_dir$repo:$branch:$build_number".repo save_repos+=("$repo:$branch:$build_number") done -disable_repos $repos_dir "${save_repos[@]}" +disable_repos "$repos_dir" "${save_repos[@]}" if [ -e /etc/profile.d/lmod.sh ]; then if ! grep "MODULEPATH=.*/usr/share/modules" /etc/profile.d/lmod.sh; then diff --git a/utils/test_memcheck.supp b/utils/test_memcheck.supp index 4f0d88fb28d..d1260ecf572 100644 --- a/utils/test_memcheck.supp +++ b/utils/test_memcheck.supp @@ -383,3 +383,9 @@ ... fun:mdb_txn_commit } +{ + go_runtime_syscall_param + Memcheck:Param + write(buf) + fun:runtime/internal/syscall.Syscall6 +}