Expanded data model for log

Expanded log capturing
Added summaries of SED document and COMBINE archive execution
Moved model resolution and modification to task loop

biosimulators_utils/_version.py

@@ -1 +1 @@
-__version__ = '0.1.31'
+__version__ = '0.1.32'

biosimulators_utils/combine/exceptions.py

@@ -0,0 +1,18 @@
+""" Exceptions for COMBINE/OMEX archives
+
+:Author: Jonathan Karr <karr@mssm.edu>
+:Date: 2021-01-12
+:Copyright: 2021, Center for Reproducible Biomedical Modeling
+:License: MIT
+"""
+
+from ..exceptions import BioSimulatorsException
+
+__all__ = [
+    'CombineArchiveExecutionError',
+]
+
+
+class CombineArchiveExecutionError(BioSimulatorsException):
+    """ Error that a SED document could not be executed """
+    pass  # pragma: no cover

biosimulators_utils/combine/exec.py

@@ -9,30 +9,36 @@
 from ..archive.io import ArchiveWriter
 from ..archive.utils import build_archive_from_paths
 from ..config import get_config
-from ..log.data_model import Status
-from ..log.utils import init_combine_archive_log
+from ..log.data_model import Status, CombineArchiveLog  # noqa: F401
+from ..log.utils import init_combine_archive_log, get_summary_combine_archive_log
 from ..plot.data_model import PlotFormat  # noqa: F401
 from ..report.data_model import DataGeneratorVariableResults, OutputResults, ReportFormat  # noqa: F401
-from ..sedml.data_model import Task, DataGeneratorVariable  # noqa: F401
+from ..sedml.data_model import (SedDocument, Task, Output, Report, DataSet, Plot2D, Curve,  # noqa: F401
+                                Plot3D, Surface, DataGeneratorVariable)
 from ..sedml.io import SedmlSimulationReader  # noqa: F401
+from ..warnings import warn
+from .exceptions import CombineArchiveExecutionError
 from .io import CombineArchiveReader
 from .utils import get_sedml_contents, get_summary_sedml_contents
 from .warnings import NoSedmlWarning
+import capturer
+import datetime
 import glob
 import os
 import tempfile
 import shutil
 import types  # noqa: F401
-import warnings
 
 __all__ = [
     'exec_sedml_docs_in_archive',
 ]
 
 
 def exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, out_dir, apply_xml_model_changes=False,
+                               sed_doc_executer_supported_features=(Task, Report, DataSet, Plot2D, Curve, Plot3D, Surface),
                                report_formats=None, plot_formats=None,
-                               bundle_outputs=None, keep_individual_outputs=None):
+                               bundle_outputs=None, keep_individual_outputs=None,
+                               sed_doc_executer_logged_features=(Task, Report, DataSet, Plot2D, Curve, Plot3D, Surface)):
     """ Execute the SED-ML files in a COMBINE/OMEX archive (execute tasks and save outputs)
 
     Args:
@@ -73,10 +79,17 @@ def sed_doc_executer(doc, working_dir, base_out_path, rel_out_path=None,
 
         apply_xml_model_changes (:obj:`bool`): if :obj:`True`, apply any model changes specified in the SED-ML files before
             calling :obj:`task_executer`.
+        sed_doc_executer_supported_features (:obj:`list` of :obj:`type`, optional): list of the types of elements that the
+            SED document executer supports. Default: tasks, reports, plots, data sets, curves, and surfaces.
         report_formats (:obj:`list` of :obj:`ReportFormat`, optional): report format (e.g., csv or h5)
         plot_formats (:obj:`list` of :obj:`PlotFormat`, optional): report format (e.g., pdf)
         bundle_outputs (:obj:`bool`, optional): if :obj:`True`, bundle outputs into archives for reports and plots
         keep_individual_outputs (:obj:`bool`, optional): if :obj:`True`, keep individual output files
+        sed_doc_executer_logged_features (:obj:`list` of :obj:`type`, optional): list of the types fo elements which that
+            the SED document executer logs. Default: tasks, reports, plots, data sets, curves, and surfaces.
+
+    Returns:
+        :obj:`CombineArchiveLog`: log
     """
     config = get_config()
 
@@ -93,6 +106,8 @@ def sed_doc_executer(doc, working_dir, base_out_path, rel_out_path=None,
     if keep_individual_outputs is None:
         keep_individual_outputs = config.KEEP_INDIVIDUAL_OUTPUTS
 
+    verbose = config.VERBOSE
+
     # create temporary directory to unpack archive
     archive_tmp_dir = tempfile.mkdtemp()
 
@@ -102,7 +117,7 @@ def sed_doc_executer(doc, working_dir, base_out_path, rel_out_path=None,
     # determine files to execute
     sedml_contents = get_sedml_contents(archive)
     if not sedml_contents:
-        warnings.warn("COMBINE/OMEX archive '{}' does not contain any executing SED-ML files".format(archive_filename), NoSedmlWarning)
+        warn("COMBINE/OMEX archive '{}' does not contain any executing SED-ML files".format(archive_filename), NoSedmlWarning)
 
     # print summary of SED documents
     print(get_summary_sedml_contents(archive, archive_tmp_dir))
@@ -112,30 +127,64 @@ def sed_doc_executer(doc, working_dir, base_out_path, rel_out_path=None,
         os.makedirs(out_dir)
 
     # initialize status and output
-    log = init_combine_archive_log(archive, archive_tmp_dir)
+    supported_features = sed_doc_executer_supported_features
+    logged_features = sed_doc_executer_logged_features
+
+    if SedDocument not in supported_features:
+        supported_features = tuple(list(supported_features) + [SedDocument])
+
+    if SedDocument not in logged_features:
+        logged_features = tuple(list(logged_features) + [SedDocument])
+
+    log = init_combine_archive_log(archive, archive_tmp_dir,
+                                   supported_features=supported_features,
+                                   logged_features=logged_features)
     log.status = Status.RUNNING
     log.out_dir = out_dir
     log.export()
+    start_time = datetime.datetime.now()
 
     # execute SED-ML files: execute tasks and save output
+    exceptions = []
     for i_content, content in enumerate(sedml_contents):
         content_filename = os.path.join(archive_tmp_dir, content.location)
         content_id = os.path.relpath(content_filename, archive_tmp_dir)
 
-        print('Executing SED-ML file {}: {}'.format(i_content, content_id))
-
-        working_dir = os.path.dirname(content_filename)
-        sed_doc_executer(content_filename,
-                         working_dir,
-                         out_dir,
-                         os.path.relpath(content_filename, archive_tmp_dir),
-                         apply_xml_model_changes=apply_xml_model_changes,
-                         report_formats=report_formats,
-                         plot_formats=plot_formats,
-                         log=log.sed_documents[content_id],
-                         indent=1)
+        print('Executing SED-ML file {}: {} ...'.format(i_content, content_id))
+
+        doc_log = log.sed_documents[content_id]
+        doc_log.status = Status.RUNNING
+        doc_log.export()
+
+        with capturer.CaptureOutput(merged=True, relay=verbose) as captured:
+            doc_start_time = datetime.datetime.now()
+            try:
+                working_dir = os.path.dirname(content_filename)
+                sed_doc_executer(content_filename,
+                                 working_dir,
+                                 out_dir,
+                                 os.path.relpath(content_filename, archive_tmp_dir),
+                                 apply_xml_model_changes=apply_xml_model_changes,
+                                 report_formats=report_formats,
+                                 plot_formats=plot_formats,
+                                 log=doc_log,
+                                 indent=1)
+                doc_log.status = Status.SUCCEEDED
+            except Exception as exception:
+                exceptions.append(exception)
+                doc_log.status = Status.FAILED
+                doc_log.exception = exception
+
+            # update status
+            doc_log.output = captured.get_bytes().decode()
+            doc_log.duration = (datetime.datetime.now() - doc_start_time).total_seconds()
+            doc_log.export()
+
+    print('')
 
     if bundle_outputs:
+        print('Bundling outputs ...')
+
         # bundle CSV files of reports into zip archive
         archive_paths = [os.path.join(out_dir, '**', '*.' + format.value) for format in report_formats if format != ReportFormat.h5]
         archive = build_archive_from_paths(archive_paths, out_dir)
@@ -149,7 +198,9 @@ def sed_doc_executer(doc, working_dir, base_out_path, rel_out_path=None,
             ArchiveWriter().run(archive, os.path.join(out_dir, config.PLOTS_PATH))
 
     # cleanup temporary files
+    print('Cleaning up ...')
     if not keep_individual_outputs:
+
         path_patterns = (
             [os.path.join(out_dir, '**', '*.' + format.value) for format in report_formats if format != ReportFormat.h5]
             + [os.path.join(out_dir, '**', '*.' + format.value) for format in plot_formats]
@@ -174,6 +225,21 @@ def sed_doc_executer(doc, working_dir, base_out_path, rel_out_path=None,
     shutil.rmtree(archive_tmp_dir)
 
     # update status
-    log.status = Status.SUCCEEDED
+    log.status = Status.FAILED if exceptions else Status.SUCCEEDED
+    log.duration = (datetime.datetime.now() - start_time).total_seconds()
     log.finalize()
     log.export()
+
+    # summarize execution
+    print('')
+    print('============= SUMMARY =============')
+    print(get_summary_combine_archive_log(log))
+
+    # raise exceptions
+    if exceptions:
+        msg = 'The COMBINE/OMEX did not execute successfully:\n\n  {}'.format(
+            '\n\n  '.join(str(exceptions).replace('\n', '\n  ') for exceptions in exceptions))
+        raise CombineArchiveExecutionError(msg)
+
+    # return log
+    return log

biosimulators_utils/combine/io.py

@@ -9,10 +9,10 @@
 from .data_model import CombineArchiveBase, CombineArchive, CombineArchiveContent, CombineArchiveContentFormat  # noqa: F401
 from ..archive.io import ArchiveReader
 from ..data_model import Person
+from ..warnings import warn
 import dateutil.parser
 import libcombine
 import os
-import warnings
 import zipfile
 
 __all__ = [
@@ -119,7 +119,7 @@ def run(cls, in_file, out_dir, try_reading_as_plain_zip_archive=True):
             if try_reading_as_plain_zip_archive:
                 try:
                     archive = CombineArchiveZipReader().run(in_file, out_dir)
-                    warnings.warn('`{}` is a plain zip archive, not a COMBINE/OMEX archive.'.format(in_file), UserWarning)
+                    warn('`{}` is a plain zip archive, not a COMBINE/OMEX archive.'.format(in_file), UserWarning)
                     return archive
                 except ValueError:
                     raise ValueError("`{}` is not a valid COMBINE/OMEX archive.".format(in_file))

biosimulators_utils/config.py

@@ -6,8 +6,11 @@
 :License: MIT
 """
 
+import enum
 import os
 
+__all__ = ['Config', 'get_config', 'Colors']
+
 
 class Config(object):
     """ Configuration
@@ -23,11 +26,12 @@ class Config(object):
         KEEP_INDIVIDUAL_OUTPUTS (:obj:`bool`): indicates whether the individual output files should be kept
         LOG_PATH (:obj:`str`): path to save the execution status of a COMBINE/OMEX archive
         BIOSIMULATORS_API_ENDPOINT (:obj:`str`): URL for BioSimulators API
+        VERBOSE (:obj:`bool`): whether to display the detailed output of the execution of each task
     """
 
     def __init__(self, ALGORITHM_SUBSTITUTION_POLICY, REPORT_FORMATS, PLOT_FORMATS,
                  H5_REPORTS_PATH, REPORTS_PATH, PLOTS_PATH, BUNDLE_OUTPUTS, KEEP_INDIVIDUAL_OUTPUTS,
-                 LOG_PATH, BIOSIMULATORS_API_ENDPOINT):
+                 LOG_PATH, BIOSIMULATORS_API_ENDPOINT, VERBOSE):
         """
         Args:
             ALGORITHM_SUBSTITUTION_POLICY (:obj:`str`): algorithm substition policy
@@ -40,6 +44,7 @@ def __init__(self, ALGORITHM_SUBSTITUTION_POLICY, REPORT_FORMATS, PLOT_FORMATS,
             KEEP_INDIVIDUAL_OUTPUTS (:obj:`bool`): indicates whether the individual output files should be kept
             LOG_PATH (:obj:`str`): path to save the execution status of a COMBINE/OMEX archive
             BIOSIMULATORS_API_ENDPOINT (:obj:`str`): URL for BioSimulators API
+            VERBOSE (:obj:`bool`): whether to display the detailed output of the execution of each task
         """
         self.ALGORITHM_SUBSTITUTION_POLICY = ALGORITHM_SUBSTITUTION_POLICY
         self.REPORT_FORMATS = REPORT_FORMATS
@@ -51,6 +56,7 @@ def __init__(self, ALGORITHM_SUBSTITUTION_POLICY, REPORT_FORMATS, PLOT_FORMATS,
         self.KEEP_INDIVIDUAL_OUTPUTS = KEEP_INDIVIDUAL_OUTPUTS
         self.LOG_PATH = LOG_PATH
         self.BIOSIMULATORS_API_ENDPOINT = BIOSIMULATORS_API_ENDPOINT
+        self.VERBOSE = VERBOSE
 
 
 def get_config():
@@ -70,4 +76,23 @@ def get_config():
         KEEP_INDIVIDUAL_OUTPUTS=os.environ.get('KEEP_INDIVIDUAL_OUTPUTS', '1').lower() in ['1', 'true'],
         LOG_PATH=os.environ.get('LOG_PATH', 'log.yml'),
         BIOSIMULATORS_API_ENDPOINT=os.environ.get('BIOSIMULATORS_API_ENDPOINT', 'https://api.biosimulators.org/'),
+        VERBOSE=os.environ.get('VERBOSE', '1').lower() in ['1', 'true'],
     )
+
+
+Colors = enum.Enum('Colors',
+                   {
+                       'queued': 'cyan',
+                       'success': 'blue',
+                       'succeeded': 'blue',
+                       'running': 'green',
+                       'pass': 'green',
+                       'passed': 'green',
+                       'failure': 'red',
+                       'failed': 'red',
+                       'skip': 'magenta',
+                       'skipped': 'magenta',
+                       'warning': 'yellow',
+                       'warned': 'yellow',
+                   },
+                   type=str)

biosimulators_utils/gh_action/data_model.py

@@ -6,13 +6,15 @@
 :License: MIT
 """
 
+from ..exceptions import BioSimulatorsException
+
 __all__ = [
     'GitHubActionCaughtError',
     'Comment',
 ]
 
 
-class GitHubActionCaughtError(Exception):
+class GitHubActionCaughtError(BioSimulatorsException):
     """ An error caught during the execution of a GitHub action """
     pass  # pragma: no cover
 

biosimulators_utils/kisao/utils.py

@@ -6,9 +6,9 @@
 :License: MIT
 """
 
+from ..warnings import warn
 from .warnings import InvalidKisaoTermIdWarning
 import re
-import warnings
 
 __all__ = ['normalize_kisao_id']
 
@@ -36,6 +36,6 @@ def normalize_kisao_id(id):
         id = 'KISAO_' + '0' * (7 - len(id)) + id
 
     if not re.match(r'KISAO_\d{7}', id):
-        warnings.warn("'{}' is likely not an id for a KiSAO term".format(unnormalized_id), InvalidKisaoTermIdWarning)
+        warn("'{}' is likely not an id for a KiSAO term".format(unnormalized_id), InvalidKisaoTermIdWarning)
 
     return id