From 91554c9cfd5754dd2fc40af7e205002d2d48ac78 Mon Sep 17 00:00:00 2001 From: Caleb Sitton Date: Tue, 2 Jul 2024 11:45:28 -0600 Subject: [PATCH 1/9] Added test type selection to run_tests --- run_tests | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/run_tests b/run_tests index 0297293c..f491a9da 100755 --- a/run_tests +++ b/run_tests @@ -10,6 +10,7 @@ else fi SCRIPT_DIR=`(cd $SCRIPT_DIRNAME; pwd)` +TEST_SET=2 # 0 for unit tests, 1 for integration tests, 2 for both # source read ravenrc script RAVEN_RC_SCRIPT=$SCRIPT_DIR/raven/scripts/read_ravenrc.sh @@ -19,6 +20,25 @@ source $RAVEN_RC_SCRIPT # set up installation manager INSTALLATION_MANAGER=$(read_ravenrc "INSTALLATION_MANAGER") +# read command-line arguments +ARGS=() +for A in "$@"; do + case $A in + --unit-tests) + TEST_SET=0 + ;; + --integration-tests) + TEST_SET=1 + ;; + --all-tests) + TEST_SET=2 + ;; + *) + ARGS+=("$A") + ;; + esac +done + echo 'Loading libraries ...' if [[ "$INSTALLATION_MANAGER" == "CONDA" ]]; @@ -57,7 +77,19 @@ echo "************************************************************************** echo echo 'Running FORCE tests ...' -$PYTHON_COMMAND $SCRIPT_DIR/raven/rook/main.py --config-file=$SCRIPT_DIR/developer_tools/rook.ini --testers-dir $RAVEN_DIR/scripts/TestHarness/testers,$RAVEN_DIR/../HERON/src/Testers "${ARGS[@]}" +case $TEST_SET in + 0) + TEST_DIR=tests/unit_tests + ;; + 1) + TEST_DIR=tests/integration_tests + ;; + 2) + TEST_DIR=tests + ;; +esac + +$PYTHON_COMMAND $SCRIPT_DIR/raven/rook/main.py --config-file=$SCRIPT_DIR/developer_tools/rook.ini --test-dir $TEST_DIR --testers-dir $RAVEN_DIR/scripts/TestHarness/testers,$RAVEN_DIR/../HERON/src/Testers "${ARGS[@]}" # store return codes individually (rc) and combined (ALL_PASS) rc=$? From f93e39bfd62cf957be91dde37a395d746eab0d31 Mon Sep 17 00:00:00 2001 From: Caleb Sitton Date: Tue, 2 Jul 2024 13:50:32 -0600 Subject: [PATCH 2/9] Code coverage implemented --- check_py_coverage.sh | 73 ++++++++++++++++++++++++++++++++++++++++++++ tests/.coveragerc | 12 ++++++++ 2 files changed, 85 insertions(+) create mode 100755 check_py_coverage.sh create mode 100644 tests/.coveragerc diff --git a/check_py_coverage.sh b/check_py_coverage.sh new file mode 100755 index 00000000..92a7766b --- /dev/null +++ b/check_py_coverage.sh @@ -0,0 +1,73 @@ +#!/bin/bash +BUILD_DIR=${BUILD_DIR:=$HOME/*/raven_libraries/build} +INSTALL_DIR=${INSTALL_DIR:=$HOME/*/*/raven_libraries} +PYTHON_CMD=${PYTHON_CMD:=python} +JOBS=${JOBS:=1} +mkdir -p $BUILD_DIR +mkdir -p $INSTALL_DIR +DOWNLOADER='curl -C - -L -O ' +SCRIPT_DIRNAME=`dirname $0` +SCRIPT_DIR=`(cd $SCRIPT_DIRNAME; pwd)` + +ORIGPYTHONPATH="$PYTHONPATH" + +update_python_path () +{ + if ls -d $INSTALL_DIR/lib/python* + then + export PYTHONPATH=`ls -d $INSTALL_DIR/lib/python*/site-packages/`:"$ORIGPYTHONPATH" + fi +} + +update_python_path +PATH=$INSTALL_DIR/bin:$PATH + +if which coverage +then + echo coverage already available, skipping building it. +else + if curl http://www.energy.gov > /dev/null + then + echo Successfully got data from the internet + else + echo Could not connect to internet + fi + + cd $BUILD_DIR + #SHA256=56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1 + $DOWNLOADER https://files.pythonhosted.org/packages/ef/05/31553dc038667012853d0a248b57987d8d70b2d67ea885605f87bcb1baba/coverage-7.5.4.tar.gz + tar -xvzf coverage-7.5.4.tar.gz + cd coverage-7.5.4 + (unset CC CXX; $PYTHON_CMD setup.py install --prefix=$INSTALL_DIR) +fi + +update_python_path + +cd $SCRIPT_DIR + +#coverage help run +SRC_DIR=`(cd src && pwd)` + +source $SCRIPT_DIR/raven/scripts/establish_conda_env.sh --quiet --load +# get display var +DISPLAY_VAR=`(echo $DISPLAY)` +# reset it +export DISPLAY= + +EXTRA="--rcfile=$SRC_DIR/../tests/.coveragerc --source=$SRC_DIR --parallel-mode" +export COVERAGE_FILE=`pwd`/.coverage + +coverage erase --rcfile="$SRC_DIR/../tests/.coveragerc" +($SRC_DIR/../run_tests "$@" --python-command="coverage run $EXTRA " || echo run_test done but some tests failed) + +#get DISPLAY BACK +DISPLAY=$DISPLAY_VAR + +## Go to the final directory and generate the html documents +cd $SCRIPT_DIR/tests/ +pwd +rm -f .cov_dirs +for FILE in `find . -name '.coverage.*'`; do dirname $FILE; done | sort | uniq > .cov_dirs +coverage combine `cat .cov_dirs` +coverage html + diff --git a/tests/.coveragerc b/tests/.coveragerc new file mode 100644 index 00000000..95f86423 --- /dev/null +++ b/tests/.coveragerc @@ -0,0 +1,12 @@ +# .coveragerc to control coverage.py + +[run] +#branch = True +parallel = True + +[report] + +ignore_errors = True + +[html] +directory = coverage_html_report From 90f9424f46c4354a76088325c9846b81aae83ea5 Mon Sep 17 00:00:00 2001 From: Caleb Sitton Date: Thu, 11 Jul 2024 10:05:51 -0600 Subject: [PATCH 3/9] Edits to code coverage --- check_py_coverage.sh | 11 ++++++----- tests/.coveragerc | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/check_py_coverage.sh b/check_py_coverage.sh index 92a7766b..0dc17ff4 100755 --- a/check_py_coverage.sh +++ b/check_py_coverage.sh @@ -1,13 +1,15 @@ #!/bin/bash -BUILD_DIR=${BUILD_DIR:=$HOME/*/raven_libraries/build} -INSTALL_DIR=${INSTALL_DIR:=$HOME/*/*/raven_libraries} +SCRIPT_DIRNAME=`dirname $0` +SCRIPT_DIR=`(cd $SCRIPT_DIRNAME; pwd)` +source $SCRIPT_DIR/raven/scripts/establish_conda_env.sh --quiet --load +RAVEN_LIBS_PATH=`conda env list | awk -v rln="$RAVEN_LIBS_NAME" '$0 ~ rln {print $NF}'` +BUILD_DIR=${BUILD_DIR:=$RAVEN_LIBS_PATH/build} +INSTALL_DIR=${INSTALL_DIR:=$RAVEN_LIBS_PATH} PYTHON_CMD=${PYTHON_CMD:=python} JOBS=${JOBS:=1} mkdir -p $BUILD_DIR mkdir -p $INSTALL_DIR DOWNLOADER='curl -C - -L -O ' -SCRIPT_DIRNAME=`dirname $0` -SCRIPT_DIR=`(cd $SCRIPT_DIRNAME; pwd)` ORIGPYTHONPATH="$PYTHONPATH" @@ -48,7 +50,6 @@ cd $SCRIPT_DIR #coverage help run SRC_DIR=`(cd src && pwd)` -source $SCRIPT_DIR/raven/scripts/establish_conda_env.sh --quiet --load # get display var DISPLAY_VAR=`(echo $DISPLAY)` # reset it diff --git a/tests/.coveragerc b/tests/.coveragerc index 95f86423..74c1dd83 100644 --- a/tests/.coveragerc +++ b/tests/.coveragerc @@ -5,6 +5,27 @@ parallel = True [report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + #def __repr__ + #if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + raise IOError + raise Exception + + # Don't complain for the things under development + pragma: under development + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: ignore_errors = True From 130ec84d678e88b217d7a840950df3ab402d81f5 Mon Sep 17 00:00:00 2001 From: Caleb Sitton Date: Thu, 11 Jul 2024 12:55:46 -0600 Subject: [PATCH 4/9] Added code coverage to run_tests github action --- .github/workflows/run_tests.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index d0deb562..c94fdbce 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -12,10 +12,18 @@ jobs: uses: actions/checkout@v3 - run: pwd - run: WD=`(cd ../../.. && pwd)` && export RAVEN_LIBS_NAME="raven_libs_"`basename $WD` && ./initalize_tests.sh - - run: source raven/scripts/establish_conda_env.sh --load && ./run_tests - - name: Archive + - run: source raven/scripts/establish_conda_env.sh --load && ./check_py_coverage.sh + - run: COV_PCT=`coverage report --format=total` + - run: echo "::notice title=Coverage Summary::Code coverage for this repository is now $COV_PCT%. See 'coverage_results' in Artifacts for details." + - name: Archive tests results uses: actions/upload-artifact@v4 if: always() with: name: tests_results path: tests + - name: Archive coverage results + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage_results + path: tests/coverage_html_report From 7a74d569e936b4ff53b56a492f72918b8d05c204 Mon Sep 17 00:00:00 2001 From: Caleb Sitton Date: Thu, 11 Jul 2024 13:44:12 -0600 Subject: [PATCH 5/9] Debugging run_tests github action --- .github/workflows/run_tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index c94fdbce..be8a05f8 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -13,8 +13,7 @@ jobs: - run: pwd - run: WD=`(cd ../../.. && pwd)` && export RAVEN_LIBS_NAME="raven_libs_"`basename $WD` && ./initalize_tests.sh - run: source raven/scripts/establish_conda_env.sh --load && ./check_py_coverage.sh - - run: COV_PCT=`coverage report --format=total` - - run: echo "::notice title=Coverage Summary::Code coverage for this repository is now $COV_PCT%. See 'coverage_results' in Artifacts for details." + - run: COV_EX=`which coverage` && COV_PCT=`$COV_EX report --format=total` && echo "::notice title=Coverage Summary::Code coverage for this repository is now $COV_PCT%. See 'coverage_results' in Artifacts for details." - name: Archive tests results uses: actions/upload-artifact@v4 if: always() From 7b1d790cc13cdd4fdc2fd155a8d8dc1a49437f78 Mon Sep 17 00:00:00 2001 From: Caleb Sitton Date: Thu, 11 Jul 2024 14:23:08 -0600 Subject: [PATCH 6/9] Debugging run_tests github action (cont) --- .github/workflows/run_tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index be8a05f8..07277f3c 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -12,8 +12,10 @@ jobs: uses: actions/checkout@v3 - run: pwd - run: WD=`(cd ../../.. && pwd)` && export RAVEN_LIBS_NAME="raven_libs_"`basename $WD` && ./initalize_tests.sh - - run: source raven/scripts/establish_conda_env.sh --load && ./check_py_coverage.sh - - run: COV_EX=`which coverage` && COV_PCT=`$COV_EX report --format=total` && echo "::notice title=Coverage Summary::Code coverage for this repository is now $COV_PCT%. See 'coverage_results' in Artifacts for details." + - run: > + source raven/scripts/establish_conda_env.sh --load && + ./check_py_coverage.sh && COV_PCT=`coverage report --format=total` && + echo "::notice title=Coverage Summary::Code coverage for this repository is now $COV_PCT%. See 'coverage_results' in Artifacts for details." - name: Archive tests results uses: actions/upload-artifact@v4 if: always() From 091e927cb820fee4e97713fb0b3072bfe33399c6 Mon Sep 17 00:00:00 2001 From: Caleb Sitton Date: Thu, 11 Jul 2024 14:38:11 -0600 Subject: [PATCH 7/9] Debugging run_tests github action (cont_2) --- .github/workflows/run_tests.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 07277f3c..882b51f1 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -13,9 +13,10 @@ jobs: - run: pwd - run: WD=`(cd ../../.. && pwd)` && export RAVEN_LIBS_NAME="raven_libs_"`basename $WD` && ./initalize_tests.sh - run: > - source raven/scripts/establish_conda_env.sh --load && - ./check_py_coverage.sh && COV_PCT=`coverage report --format=total` && - echo "::notice title=Coverage Summary::Code coverage for this repository is now $COV_PCT%. See 'coverage_results' in Artifacts for details." + source raven/scripts/establish_conda_env.sh --load && + ./check_py_coverage.sh && + COV_PCT=`coverage report --format=total` && + echo "::notice title=Coverage Summary::Code coverage for this repository is now $COV_PCT%. See 'coverage_results' in Artifacts for details." - name: Archive tests results uses: actions/upload-artifact@v4 if: always() From ad2472deaafadaba2ae0cb2d9b97f6d02a0da412 Mon Sep 17 00:00:00 2001 From: Caleb Sitton Date: Tue, 16 Jul 2024 15:45:25 -0600 Subject: [PATCH 8/9] Edits to code coverage setup --- .github/workflows/run_tests.yml | 2 +- check_py_coverage.sh | 14 ++++++-------- run_tests | 10 ++++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 882b51f1..f699d854 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -12,7 +12,7 @@ jobs: uses: actions/checkout@v3 - run: pwd - run: WD=`(cd ../../.. && pwd)` && export RAVEN_LIBS_NAME="raven_libs_"`basename $WD` && ./initalize_tests.sh - - run: > + - run: > # The overhead time added by checking coverage is currently about 19-23%. Reducing the frequency of coverage checks may be preferable if this increases. source raven/scripts/establish_conda_env.sh --load && ./check_py_coverage.sh && COV_PCT=`coverage report --format=total` && diff --git a/check_py_coverage.sh b/check_py_coverage.sh index 0dc17ff4..bebeff6d 100755 --- a/check_py_coverage.sh +++ b/check_py_coverage.sh @@ -55,20 +55,18 @@ DISPLAY_VAR=`(echo $DISPLAY)` # reset it export DISPLAY= -EXTRA="--rcfile=$SRC_DIR/../tests/.coveragerc --source=$SRC_DIR --parallel-mode" +export COVERAGE_RCFILE="$SRC_DIR/../tests/.coveragerc" # all coverage commands should automatically reference this file now +EXTRA="--source=$SRC_DIR --parallel-mode" export COVERAGE_FILE=`pwd`/.coverage -coverage erase --rcfile="$SRC_DIR/../tests/.coveragerc" -($SRC_DIR/../run_tests "$@" --python-command="coverage run $EXTRA " || echo run_test done but some tests failed) +coverage erase +($SRC_DIR/../run_tests "$@" --python-command="coverage run $EXTRA " || echo run_tests done but some tests failed) #get DISPLAY BACK DISPLAY=$DISPLAY_VAR -## Go to the final directory and generate the html documents -cd $SCRIPT_DIR/tests/ +## Prepare data and generate the html documents pwd -rm -f .cov_dirs -for FILE in `find . -name '.coverage.*'`; do dirname $FILE; done | sort | uniq > .cov_dirs -coverage combine `cat .cov_dirs` +coverage combine coverage html diff --git a/run_tests b/run_tests index f491a9da..3b4d3d24 100755 --- a/run_tests +++ b/run_tests @@ -75,17 +75,19 @@ FAILED=() echo echo "********************************************************************************" echo -echo 'Running FORCE tests ...' case $TEST_SET in 0) - TEST_DIR=tests/unit_tests + TEST_DIR=$SCRIPT_DIR/tests/unit_tests + echo Running FORCE unit tests from $TEST_DIR directory ... ;; 1) - TEST_DIR=tests/integration_tests + TEST_DIR=$SCRIPT_DIR/tests/integration_tests + echo Running FORCE integration tests from $TEST_DIR directory ... ;; 2) - TEST_DIR=tests + TEST_DIR=$SCRIPT_DIR/tests + echo Running FORCE unit and integration tests from $TEST_DIR directory ... ;; esac From 88f0fc52e2da001044d4d099d0414a77ff3fe43d Mon Sep 17 00:00:00 2001 From: Caleb Sitton Date: Tue, 16 Jul 2024 16:05:06 -0600 Subject: [PATCH 9/9] Fixed bug with coverage results location --- tests/.coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/.coveragerc b/tests/.coveragerc index 74c1dd83..eeef7918 100644 --- a/tests/.coveragerc +++ b/tests/.coveragerc @@ -30,4 +30,4 @@ exclude_lines = ignore_errors = True [html] -directory = coverage_html_report +directory = tests/coverage_html_report