Merge pull request idaholab#377 from caleb-sitton-inl/code-coverage

Code coverage

check_py_coverage.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+SCRIPT_DIRNAME=`dirname $0`
+SCRIPT_DIR=`(cd $SCRIPT_DIRNAME; pwd)`
+RAVEN_DIR=`python -c 'from src._utils import get_raven_loc; print(get_raven_loc())'`
+source $RAVEN_DIR/scripts/establish_conda_env.sh --quiet --load
+RAVEN_LIBS_PATH=`conda env list | awk -v rln="$RAVEN_LIBS_NAME" '$0 ~ rln {print $NF}'`
+BUILD_DIR=${BUILD_DIR:=$RAVEN_LIBS_PATH/build}
+INSTALL_DIR=${INSTALL_DIR:=$RAVEN_LIBS_PATH}
+PYTHON_CMD=${PYTHON_CMD:=python}
+JOBS=${JOBS:=1}
+mkdir -p $BUILD_DIR
+mkdir -p $INSTALL_DIR
+DOWNLOADER='curl -C - -L -O '
+
+ORIGPYTHONPATH="$PYTHONPATH"
+
+update_python_path ()
+{
+    if ls -d $INSTALL_DIR/lib/python*
+    then
+        export PYTHONPATH=`ls -d $INSTALL_DIR/lib/python*/site-packages/`:"$ORIGPYTHONPATH"
+    fi
+}
+
+update_python_path
+PATH=$INSTALL_DIR/bin:$PATH
+
+if which coverage
+then
+    echo coverage already available, skipping building it.
+else
+    if curl http://www.energy.gov > /dev/null
+    then
+       echo Successfully got data from the internet
+    else
+       echo Could not connect to internet
+    fi
+
+    cd $BUILD_DIR
+    #SHA256=56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1
+    $DOWNLOADER https://files.pythonhosted.org/packages/ef/05/31553dc038667012853d0a248b57987d8d70b2d67ea885605f87bcb1baba/coverage-7.5.4.tar.gz
+    tar -xvzf coverage-7.5.4.tar.gz
+    cd coverage-7.5.4
+    (unset CC CXX; $PYTHON_CMD setup.py install --prefix=$INSTALL_DIR)
+fi
+
+update_python_path
+
+cd $SCRIPT_DIR
+
+#coverage help run
+SRC_DIR=`(cd src && pwd)`
+
+# get display var
+DISPLAY_VAR=`(echo $DISPLAY)`
+# reset it
+export DISPLAY=
+
+export COVERAGE_RCFILE="$SRC_DIR/../tests/.coveragerc"
+SOURCE_DIRS=($SRC_DIR,$SRC_DIR/../templates/)
+OMIT_FILES=($SRC_DIR/dispatch/twin_pyomo_test.py,$SRC_DIR/dispatch/twin_pyomo_test_rte.py,$SRC_DIR/dispatch/twin_pyomo_limited_ramp.py,$SRC_DIR/ArmaBypass.py)
+EXTRA="--source=${SOURCE_DIRS[@]} --omit=${OMIT_FILES[@]} --parallel-mode "
+export COVERAGE_FILE=`pwd`/.coverage
+
+coverage erase
+($RAVEN_DIR/run_tests "$@" --re=HERON/tests --python-command="coverage run $EXTRA " || echo run_tests done but some tests failed)
+
+#get DISPLAY BACK
+DISPLAY=$DISPLAY_VAR
+
+## Prepare data and generate the html documents
+coverage combine
+coverage html
+

heron

@@ -33,11 +33,14 @@ source $RAVEN_DIR/scripts/establish_conda_env.sh --quiet
 declare -a ARGS
 while test $# -gt 0
 do
-  # right now we don't have any keyword arguments for this script, but leave this for now
   case "$1" in
+    --python-command=*)
+      PYTHON_COMMAND="${1#*=}"
+      ;;  
     *)
       # otherwise, pass through arguments to main.py
       ARGS[${#ARGS[@]}]="$1"
+      ;;
   esac
   shift
 done

src/Cases.py

@@ -673,6 +673,8 @@ def __init__(self, run_dir, **kwargs):
         'expectedValue': None,
         'median': None}
 
+    self._python_command_for_raven = None # python command to use for raven executable
+
     # clean up location
     self.run_dir = os.path.abspath(os.path.expanduser(self.run_dir))
 
@@ -1324,6 +1326,23 @@ def get_opt_strategy(self):
     """
     return self._optimization_strategy
 
+  def get_py_cmd_for_raven(self):
+    """
+      Accessor
+      @ In, None
+      @ Out, py_cmd_for_raven, str, custom python command for running raven (if set)
+    """
+    return self._python_command_for_raven
+
+  def set_py_cmd_for_raven(self, py_cmd):
+    """
+      Mutator
+      @ In, py_cmd, str, custom python command for running raven
+      @ Out, None
+    """
+    self._python_command_for_raven = py_cmd
+    return
+
   @property
   def npv_target(self):
     """

src/Testers/HeronIntegrationTester.py

@@ -93,7 +93,9 @@ def get_heron_command(self, cmd):
     # Windows is a little different with bash scripts
     if platform.system() == 'Windows':
       cmd += ' bash.exe '
-    cmd += f' {self.heron_driver} {heron_inp}'
+    python = self._get_python_command()
+    # python-command is for running HERON; python_command_for_raven is for running RAVEN inner
+    cmd += f' {self.heron_driver} --python-command="{python}" --python_command_for_raven="{python}" {heron_inp}'
     return cmd, heron_inp
 
   def get_raven_command(self, cmd, heron_inp):

src/main.py

@@ -99,16 +99,20 @@ def plot_resource_graph(self) -> None:
       img_path = os.path.join(self._input_dir, 'network.png')
       graph.save(img_path)
 
-  def create_raven_workflow(self, case=None):
+  def create_raven_workflow(self, case=None, python_cmd_raven=None):
     """
       Loads, modifies, and writes a RAVEN template workflow based on the Case.
       @ In, case, Cases.Case, optional, case to run (defaults to self._case)
+      @ In, python_cmd_raven, string, optional, custom python command to use for running RAVEN
+          Driving use case is specifying that RAVEN be run with code coverage
       @ Out, None
     """
     if case is None:
       case = self._case
     # let the case do the work
     assert case is not None
+    if python_cmd_raven:
+      case.set_py_cmd_for_raven(python_cmd_raven)
     case.write_workflows(self._components, self._sources, self._input_dir)
 
   def run_moped_workflow(self, case=None, components=None, sources=None):
@@ -163,6 +167,7 @@ def main():
   parser = argparse.ArgumentParser(description='Holistic Energy Resource Optimization Network (HERON)')
   parser.add_argument('xml_input_file', nargs='?', default="", help='HERON XML input file')
   parser.add_argument('--definition', action="store_true", dest="definition", help='HERON input file definition compatible with the NEAMS Workbench')
+  parser.add_argument('--python_command_for_raven', dest='python_cmd_raven', help='Custom python command for running RAVEN inner')
   args = parser.parse_args()
 
   sim = HERON()
@@ -181,7 +186,10 @@ def main():
   sim.plot_resource_graph()
 
   if sim._case._workflow == 'standard':
-    sim.create_raven_workflow()
+    if args.python_cmd_raven is not None:
+      sim.create_raven_workflow(python_cmd_raven=args.python_cmd_raven)
+    else:
+      sim.create_raven_workflow()
   elif sim._case._workflow == 'MOPED':
     sim.run_moped_workflow()
   elif sim._case._workflow == 'DISPATCHES':

templates/template_driver.py

@@ -480,6 +480,11 @@ def _modify_outer_models(self, template, case, components, sources):
       raven_exec.text = "raven_framework"
     else:
       raise RuntimeError("raven_framework not in PATH and not at "+raven_exec_guess)
+    # custom python command for running raven (for example, "coverage run")
+    if case.get_py_cmd_for_raven() is not None:
+      attribs = {'type': 'prepend', 'arg': case.get_py_cmd_for_raven()}
+      new = xmlUtils.newNode('clargs', attrib=attribs)
+      raven.append(new)
     # conversion script
     conv = raven.find('conversion').find('input')
     conv.attrib['source'] = '../write_inner.py'

tests/.coveragerc

@@ -0,0 +1,32 @@
+# .coveragerc to control coverage.py
+[run]
+#branch = True
+parallel = True
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    #def __repr__
+    #if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+    raise IOError
+    raise Exception
+
+    # Don't complain for the things under development
+    pragma: under development
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+
+ignore_errors = True
+
+[html]
+directory = tests/coverage_html_report

tests/README.md

@@ -0,0 +1,11 @@
+# Code Coverage
+## Coverage source
+The "source" for code coverage defines a list of files over which code coverage is checked. This is defined in the `HERON/check_py_coverage.sh` script. By default when the source directory is specified, coverage.py measures coverage over all files in the source directory(ies) ending with .py, .pyw, .pyo, or .pyc that have typical punctuation. It also measures all files in subdirectories that also include an `__init__.py` file. For details see https://coverage.readthedocs.io/en/7.5.4/source.html#source
+
+HERON code coverage is currently set up to run all files in the `HERON/src/` directory as well as in the `HERON/templates/` directory (provided the limitations listed above). Exceptions, which are in these directories but not covered, are listed as omitted files and directories in `HERON/check_py_coverage.sh`. Currently this list is comprised of the following files:
+- `HERON/src/ARMABypass.py`
+- `HERON/src/dispatch/twin_pyomo_test.py`
+- `HERON/src/dispatch/twin_pyomo_test_rte.py`
+- `HERON/src/dispatch/twin_pyomo_limited_ramp.py`
+
+Note additionally that files in some subdirectories of `HERON/src` are omitted automatically by coverage.py because those subdirectories lack an `__init__.py` file. An example is the `HERON/src/Testers/` directory.