Merge pull request #22 from caleb-sitton-inl/code-coverage

Code coverage with github action - final

.github/workflows/run_tests.yml

@@ -12,10 +12,20 @@ jobs:
         uses: actions/checkout@v3
       - run: pwd
       - run: WD=`(cd ../../.. && pwd)` && export RAVEN_LIBS_NAME="raven_libs_"`basename $WD` && ./initalize_tests.sh
-      - run: source raven/scripts/establish_conda_env.sh --load && ./run_tests
-      - name: Archive
+      - run: > # The overhead time added by checking coverage is currently about 19-23%. Reducing the frequency of coverage checks may be preferable if this increases.
+          source raven/scripts/establish_conda_env.sh --load &&
+          ./check_py_coverage.sh &&
+          COV_PCT=`coverage report --format=total` &&
+          echo "::notice title=Coverage Summary::Code coverage for this repository is now $COV_PCT%. See 'coverage_results' in Artifacts for details."
+      - name: Archive tests results
         uses: actions/upload-artifact@v4
         if: always()
         with:
           name: tests_results
           path: tests
+      - name: Archive coverage results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: coverage_results
+          path: tests/coverage_html_report

check_py_coverage.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+SCRIPT_DIRNAME=`dirname $0`
+SCRIPT_DIR=`(cd $SCRIPT_DIRNAME; pwd)`
+source $SCRIPT_DIR/raven/scripts/establish_conda_env.sh --quiet --load
+RAVEN_LIBS_PATH=`conda env list | awk -v rln="$RAVEN_LIBS_NAME" '$0 ~ rln {print $NF}'`
+BUILD_DIR=${BUILD_DIR:=$RAVEN_LIBS_PATH/build}
+INSTALL_DIR=${INSTALL_DIR:=$RAVEN_LIBS_PATH}
+PYTHON_CMD=${PYTHON_CMD:=python}
+JOBS=${JOBS:=1}
+mkdir -p $BUILD_DIR
+mkdir -p $INSTALL_DIR
+DOWNLOADER='curl -C - -L -O '
+
+ORIGPYTHONPATH="$PYTHONPATH"
+
+update_python_path ()
+{
+    if ls -d $INSTALL_DIR/lib/python*
+    then
+        export PYTHONPATH=`ls -d $INSTALL_DIR/lib/python*/site-packages/`:"$ORIGPYTHONPATH"
+    fi
+}
+
+update_python_path
+PATH=$INSTALL_DIR/bin:$PATH
+
+if which coverage
+then
+    echo coverage already available, skipping building it.
+else
+    if curl http://www.energy.gov > /dev/null
+    then
+       echo Successfully got data from the internet
+    else
+       echo Could not connect to internet
+    fi
+
+    cd $BUILD_DIR
+    #SHA256=56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1
+    $DOWNLOADER https://files.pythonhosted.org/packages/ef/05/31553dc038667012853d0a248b57987d8d70b2d67ea885605f87bcb1baba/coverage-7.5.4.tar.gz
+    tar -xvzf coverage-7.5.4.tar.gz
+    cd coverage-7.5.4
+    (unset CC CXX; $PYTHON_CMD setup.py install --prefix=$INSTALL_DIR)
+fi
+
+update_python_path
+
+cd $SCRIPT_DIR
+
+#coverage help run
+SRC_DIR=`(cd src && pwd)`
+
+# get display var
+DISPLAY_VAR=`(echo $DISPLAY)`
+# reset it
+export DISPLAY=
+
+export COVERAGE_RCFILE="$SRC_DIR/../tests/.coveragerc" # all coverage commands should automatically reference this file now
+EXTRA="--source=$SRC_DIR --parallel-mode"
+export COVERAGE_FILE=`pwd`/.coverage
+
+coverage erase
+($SRC_DIR/../run_tests "$@" --python-command="coverage run $EXTRA " || echo run_tests done but some tests failed)
+
+#get DISPLAY BACK
+DISPLAY=$DISPLAY_VAR
+
+## Prepare data and generate the html documents
+pwd
+coverage combine
+coverage html
+

run_tests

@@ -10,6 +10,7 @@ else
 fi
 SCRIPT_DIR=`(cd $SCRIPT_DIRNAME; pwd)`
 
+TEST_SET=2 # 0 for unit tests, 1 for integration tests, 2 for both
 
 # source read ravenrc script
 RAVEN_RC_SCRIPT=$SCRIPT_DIR/raven/scripts/read_ravenrc.sh
@@ -19,6 +20,25 @@ source $RAVEN_RC_SCRIPT
 # set up installation manager
 INSTALLATION_MANAGER=$(read_ravenrc "INSTALLATION_MANAGER")
 
+# read command-line arguments
+ARGS=()
+for A in "$@"; do
+    case $A in
+        --unit-tests)
+            TEST_SET=0
+            ;;
+        --integration-tests)
+            TEST_SET=1
+            ;;
+        --all-tests)
+            TEST_SET=2
+            ;;
+        *)
+            ARGS+=("$A")
+            ;;
+    esac
+done
+
 
 echo 'Loading libraries ...'
 if  [[ "$INSTALLATION_MANAGER" == "CONDA" ]];
@@ -55,9 +75,23 @@ FAILED=()
 echo
 echo "********************************************************************************"
 echo
-echo 'Running FORCE tests ...'
 
-$PYTHON_COMMAND $SCRIPT_DIR/raven/rook/main.py --config-file=$SCRIPT_DIR/developer_tools/rook.ini --testers-dir $RAVEN_DIR/scripts/TestHarness/testers,$RAVEN_DIR/../HERON/src/Testers "${ARGS[@]}"
+case $TEST_SET in
+    0)
+        TEST_DIR=$SCRIPT_DIR/tests/unit_tests
+        echo Running FORCE unit tests from $TEST_DIR directory ...
+        ;;
+    1)
+        TEST_DIR=$SCRIPT_DIR/tests/integration_tests
+        echo Running FORCE integration tests from $TEST_DIR directory ...
+        ;;
+    2)
+        TEST_DIR=$SCRIPT_DIR/tests
+        echo Running FORCE unit and integration tests from $TEST_DIR directory ...
+        ;;
+esac
+
+$PYTHON_COMMAND $SCRIPT_DIR/raven/rook/main.py --config-file=$SCRIPT_DIR/developer_tools/rook.ini --test-dir $TEST_DIR --testers-dir $RAVEN_DIR/scripts/TestHarness/testers,$RAVEN_DIR/../HERON/src/Testers "${ARGS[@]}"
 
 # store return codes individually (rc) and combined (ALL_PASS)
 rc=$?

tests/.coveragerc

@@ -0,0 +1,33 @@
+# .coveragerc to control coverage.py
+
+[run]
+#branch = True
+parallel = True
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    #def __repr__
+    #if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+    raise IOError
+    raise Exception
+
+    # Don't complain for the things under development
+    pragma: under development
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+
+ignore_errors = True
+
+[html]
+directory = tests/coverage_html_report