Merge pull request #228 from pepkit/summarizer-updates

v0.12.6

docs/changelog.md

@@ -2,6 +2,22 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. 
 
+## [0.12.6] -- 2020-02-21
+### Added
+- possibility to execute library module as a script: `python -m looper ...`
+
+### Changed
+- in the summary page account for missing values when plotting; the value is disregarded in such a case and plot is still created
+- show 50 rows in the summary table
+- make links to the summary page relative
+- long entries in the sample stats table are truncated with an option to see original value in a popover
+
+### Fixed
+- inactive jQuery dependent components in the status page
+- project objects layout in the summary index page
+- inactivation of popovers after Bootstrap Table events
+- non-homogeneous status flags appearance
+
 ## [0.12.5] -- 2019-12-13
 ### Changed
 - reduce verbosity of missing options; [Issue 174](https://github.com/pepkit/looper/issues/174)

looper/__init__.py

@@ -71,7 +71,7 @@ def build_parser():
     additional_description = "For subcommand-specific options, type: '%(prog)s <subcommand> -h'"
     additional_description += "\nhttps://github.com/pepkit/looper"
 
-    parser = VersionInHelpParser(description=banner, epilog=additional_description, version=__version__)
+    parser = VersionInHelpParser(prog="looper", description=banner, epilog=additional_description, version=__version__)
 
     # Logging control
     parser.add_argument(

looper/__main__.py

@@ -0,0 +1,9 @@
+from .looper import main
+import sys
+
+if __name__ == '__main__':
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        print("Program canceled by user!")
+        sys.exit(1)

looper/_version.py

@@ -1 +1 @@
-__version__ = "0.12.5"
+__version__ = "0.12.6"

looper/const.py

@@ -1,35 +1,55 @@
 """ Shared project constants """
 
-__author__ = "Vince Reuter"
-__email__ = "vreuter@virginia.edu"
+__author__ = "Databio lab"
+__email__ = "nathan@code.databio.org"
 
 
-__all__ = ["APPEARANCE_BY_FLAG", "NO_DATA_PLACEHOLDER", "OUTKEY",
+__all__ = ["BUTTON_APPEARANCE_BY_FLAG", "TABLE_APPEARANCE_BY_FLAG", "NO_DATA_PLACEHOLDER", "OUTKEY",
            "PIPELINE_INTERFACES_KEY", "PIPELINE_REQUIREMENTS_KEY",
            "RESULTS_SUBDIR_KEY", "SUBMISSION_SUBDIR_KEY", "TEMPLATES_DIRNAME"]
 
 APPEARANCE_BY_FLAG = {
     "completed": {
-        "button_class": "table-success",
+        "button_class": "{type}-success",
         "flag": "Completed"
     },
     "running": {
-        "button_class": "table-primary",
+        "button_class": "{type}-primary",
         "flag": "Running"
     },
     "failed": {
-        "button_class": "table-danger",
+        "button_class": "{type}-danger",
         "flag": "Failed"
     },
     "parital": {
-        "button_class": "table-warning",
+        "button_class": "{type}-warning",
         "flag": "Partial"
     },
     "waiting": {
-        "button_class": "table-info",
+        "button_class": "{type}-info",
         "flag": "Waiting"
     }
 }
+
+
+def _get_apperance_dict(type, templ=APPEARANCE_BY_FLAG):
+    """
+    Based on the type of the HTML element provided construct the appearence mapping using the template
+
+    :param dict templ: appearance templete to populate
+    :param str type: type of HTML element to populate template with
+    :return dict: populated appearance template
+    """
+    from copy import deepcopy
+    ret = deepcopy(templ)
+    for flag, app_dict in ret.items():
+        for key, app in app_dict.items():
+            ret[flag][key] = ret[flag][key].format(type=type)
+    return ret
+
+
+TABLE_APPEARANCE_BY_FLAG = _get_apperance_dict("table")
+BUTTON_APPEARANCE_BY_FLAG = _get_apperance_dict("btn btn")
 NO_DATA_PLACEHOLDER = "NA"
 PIPELINE_INTERFACES_KEY = "pipeline_interfaces"
 PIPELINE_REQUIREMENTS_KEY = "required_executables"

looper/html_reports.py

@@ -10,7 +10,7 @@
 from warnings import warn
 from datetime import timedelta
 from ._version import __version__ as v
-from .const import TEMPLATES_DIRNAME, APPEARANCE_BY_FLAG, NO_DATA_PLACEHOLDER, IMAGE_EXTS, PROFILE_COLNAMES
+from .const import TEMPLATES_DIRNAME, BUTTON_APPEARANCE_BY_FLAG, TABLE_APPEARANCE_BY_FLAG, NO_DATA_PLACEHOLDER, IMAGE_EXTS, PROFILE_COLNAMES
 from copy import copy as cp
 _LOGGER = logging.getLogger("looper")
 
@@ -29,15 +29,18 @@ def __init__(self, prj):
         self.j_env = get_jinja_env()
         self.reports_dir = get_reports_dir(self.prj)
         self.index_html_path = get_index_html_path(self.prj)
+        self.index_html_filename = os.path.basename(self.index_html_path)
         _LOGGER.debug("Reports dir: {}".format(self.reports_dir))
 
     def __call__(self, objs, stats, columns):
         """ Do the work of the subcommand/program. """
         # Generate HTML report
-        navbar = self.create_navbar(self.create_navbar_links(objs=objs, stats=stats, wd=self.prj.metadata.output_dir))
-        navbar_reports = self.create_navbar(self.create_navbar_links(objs=objs, stats=stats, wd=self.reports_dir))
-        index_html_path = self.create_index_html(objs, stats, columns, navbar=navbar, navbar_reports=navbar_reports,
-                                                 footer=self.create_footer())
+        navbar = self.create_navbar(self.create_navbar_links(objs=objs, stats=stats, wd=self.prj.metadata.output_dir),
+                                    self.index_html_filename)
+        navbar_reports = self.create_navbar(self.create_navbar_links(objs=objs, stats=stats, wd=self.reports_dir),
+                                            os.path.join("..", self.index_html_filename))
+        index_html_path = self.create_index_html(objs, stats, columns, footer=self.create_footer(), navbar=navbar,
+                                                 navbar_reports=navbar_reports)
         return index_html_path
 
     def create_object_parent_html(self, objs, navbar, footer):
@@ -97,14 +100,14 @@ def create_sample_parent_html(self, navbar, footer):
         template_vars = dict(navbar=navbar, footer=footer, labels=labels, pages=pages, header="Samples")
         return render_jinja_template("navbar_list_parent.html", self.j_env, template_vars)
 
-    def create_navbar(self, navbar_links):
+    def create_navbar(self, navbar_links, index_html_relpath):
         """
         Creates the navbar using the privided links
 
         :param str navbar_links: HTML list of links to be inserted into a navbar
         :return str: navbar HTML
         """
-        template_vars = dict(navbar_links=navbar_links, index_html=self.index_html_path)
+        template_vars = dict(navbar_links=navbar_links, index_html=index_html_relpath)
         return render_jinja_template("navbar.html", self.j_env, template_vars)
 
     def create_footer(self):
@@ -184,39 +187,35 @@ def create_object_html(self, single_object, navbar, footer):
             # even though it's always one element, loop to extract the data
             current_name = str(key)
             filename = current_name + ".html"
-        object_path = os.path.join(self.reports_dir, filename.replace(' ', '_').lower())
+        html_page_path = os.path.join(self.reports_dir, filename.replace(' ', '_').lower())
 
-        if not os.path.exists(os.path.dirname(object_path)):
-            os.makedirs(os.path.dirname(object_path))
+        if not os.path.exists(os.path.dirname(html_page_path)):
+            os.makedirs(os.path.dirname(html_page_path))
 
         links = []
         figures = []
         warnings = []
         for i, row in single_object.iterrows():
             # Set the PATH to a page for the sample. Catch any errors.
             try:
-                page_path = os.path.join(self.prj.results_folder, row['sample_name'], row['filename'])
+                object_path = os.path.join(self.prj.results_folder, row['sample_name'], row['filename'])
+                object_relpath = os.path.relpath(object_path, self.reports_dir)
             except AttributeError:
-                err_msg = ("Sample: {} | " + "Missing valid page path for: {}")
+                err_msg = ("Sample: {} | " + "Missing valid object path for: {}")
                 # Report the sample that fails, if that information exists
                 if str(row['sample_name']) and str(row['filename']):
                     _LOGGER.warn(err_msg.format(row['sample_name'], row['filename']))
                 else:
                     _LOGGER.warn(err_msg.format("Unknown sample"))
-                page_path = ""
-            if not page_path.strip():
-                page_relpath = os.path.relpath(page_path, self.reports_dir)
-            else:
-                page_relpath = ""
+                object_relpath = ""
 
             # Set the PATH to the image/file. Catch any errors.
             # Check if the object is an HTML document
             if not str(row['anchor_image']).lower().endswith(IMAGE_EXTS):
-                image_path = page_path
+                image_path = object_path
             else:
                 try:
-                    image_path = os.path.join(self.prj.results_folder,
-                                              row['sample_name'], row['anchor_image'])
+                    image_path = os.path.join(self.prj.results_folder, row['sample_name'], row['anchor_image'])
                 except AttributeError:
                     _LOGGER.warn(str(row))
                     err_msg = ("Sample: {} | " + "Missing valid image path for: {}")
@@ -228,12 +227,12 @@ def create_object_html(self, single_object, navbar, footer):
                     image_path = ""
 
             # Check for the presence of both the file and thumbnail
-            if os.path.isfile(image_path) and os.path.isfile(page_path):
+            if os.path.isfile(image_path) and os.path.isfile(object_path):
                 image_relpath = os.path.relpath(image_path, self.reports_dir)
                 # If the object has a valid image, use it!
                 _LOGGER.debug("Checking image path: {}".format(image_path))
                 if str(image_path).lower().endswith(IMAGE_EXTS):
-                    figures.append([page_relpath, str(row['sample_name']), image_relpath])
+                    figures.append([object_relpath, str(row['sample_name']), image_relpath])
                 # Or if that "image" is not an image, treat it as a link
                 elif not str(image_path).lower().endswith(IMAGE_EXTS):
                     _LOGGER.debug("Got link")
@@ -247,7 +246,7 @@ def create_object_html(self, single_object, navbar, footer):
             _LOGGER.debug(filename.replace(' ', '_').lower() +
                           " nonexistent files: " + ','.join(str(x) for x in warnings))
         template_vars = dict(navbar=navbar, footer=footer, name=current_name, figures=figures, links=links)
-        save_html(object_path, render_jinja_template("object.html", self.j_env, args=template_vars))
+        save_html(html_page_path, render_jinja_template("object.html", self.j_env, args=template_vars))
 
     def create_sample_html(self, objs, sample_name, sample_stats, navbar, footer):
         """
@@ -269,20 +268,6 @@ def create_sample_html(self, objs, sample_name, sample_stats, navbar, footer):
         if not os.path.exists(os.path.dirname(html_page)):
             os.makedirs(os.path.dirname(html_page))
         sample_dir = os.path.join(self.prj.results_folder, sample_name)
-        button_appearance_by_flag = {
-            "completed": {
-                "button_class": "btn btn-success",
-                "flag": "Completed"
-            },
-            "running": {
-                "button_class": "btn btn-warning",
-                "flag": "Running"
-            },
-            "failed": {
-                "button_class": "btn btn-danger",
-                "flag": "Failed"
-            }
-        }
         if os.path.exists(sample_dir):
             if single_sample.empty:
                 # When there is no objects.tsv file, search for the
@@ -306,15 +291,15 @@ def create_sample_html(self, objs, sample_name, sample_stats, navbar, footer):
             log_file_path = _get_relpath_to_file(
                 log_name, sample_name, self.prj.results_folder, self.reports_dir)
             if not flag:
-                button_class = "btn btn-danger"
+                button_class = "btn btn-secondary"
                 flag = "Missing"
             elif len(flag) > 1:
-                button_class = "btn btn-warning"
+                button_class = "btn btn-secondary"
                 flag = "Multiple"
             else:
                 flag = flag[0]
                 try:
-                    flag_dict = button_appearance_by_flag[flag]
+                    flag_dict = BUTTON_APPEARANCE_BY_FLAG[flag]
                 except KeyError:
                     button_class = "btn btn-secondary"
                     flag = "Unknown"
@@ -396,6 +381,7 @@ def create_project_objects(self):
 
         # For each protocol report the project summarizers' results
         for protocol in set(all_protocols):
+            _LOGGER.debug("Creating project objects for protocol:{}".format(protocol))
             figures = []
             links = []
             warnings = []
@@ -798,7 +784,7 @@ def create_status_table(prj, final=True):
             else:
                 flag = flag[0]
                 try:
-                    flag_dict = APPEARANCE_BY_FLAG[flag]
+                    flag_dict = TABLE_APPEARANCE_BY_FLAG[flag]
                 except KeyError:
                     button_class = "table-secondary"
                     flag = "Unknown"

looper/jinja_templates/footer.html

@@ -1,3 +1,3 @@
 <div class="wrapper">
 Generated with <code>looper v{{ version }}</code>
-<span style="float:right;">&copy; 2018 <a href="http://databio.org/" target="_new">Sheffield Computational Biology Lab</a></span>
+<span style="float:right;">&copy; 2018-2020 <a href="http://databio.org/" target="_new">Sheffield Computational Biology Lab</a></span>

looper/jinja_templates/footer_index.html

@@ -2,12 +2,13 @@
 <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
 <script type="text/javascript">
     var plotDict = {}; // variable containing column values (col_name:string -> col_values:list)
+    var nameDict = {} // variable sample names (col_name:string -> sample_names:string)
     var statsTable = {{ stats_json }}; // Jinja template variable
     // make a list of the first column values, which are the names of the samples
-    var sample_names = [];
+    var all_names = [];
     var j = 1;
     while(typeof(statsTable[0][j]) !== "undefined"){
-        sample_names.push(statsTable[0][j]);
+        all_names.push(statsTable[0][j]);
         j++;
     }
     // populate the values of each column
@@ -21,23 +22,27 @@
         }
         var colName = currCol[0];
         var colValues = [];
-        var j = 1;
-        while(typeof(currCol[j]) !== "undefined" && !Array.isArray(currCol[j].match(re))){
-            colValues.push(parseFloat(currCol[j]));
-            j++;
+        var sample_names = [];
+        var arrayLength = Object.keys(currCol).length;
+        for (var j = 1; j < arrayLength; j++) {
+            if ((typeof(currCol[j]) !== "undefined" && currCol[j] !== null && !Array.isArray(currCol[j].match(re)))){
+                colValues.push(parseFloat(currCol[j]));
+                sample_names.push(all_names[j-1]);
+            }
         }
         console.log("colValues: " + colValues)
         if(colValues.length > 0){
             console.log("skipped")
             plotDict[colName] = colValues;
+            nameDict[colName] = sample_names;
         }
         i++;
     }
 
     function graphCol(colname){
       /* Creates a graph using Plotly.js */
       var data = [{
-        x: sample_names,
+        x: nameDict[colname],
         y: plotDict[colname],
         type: 'bar',
       }];
@@ -47,11 +52,13 @@
       chart = document.getElementById('charts');
       Plotly.newPlot(chart, data, layout);
     }
+
     function displayPlottableColumns(){
         for(var col in plotDict){
             $("#plot-cols").append("<li class='list-group-item'><a href='#' title='Click to visualize this column' onclick='graphCol(\"" + col +
                 "\");return false;'>" + col+ "</a></li>");
         }
     }
+
     displayPlottableColumns();
 </script>

looper/jinja_templates/index.html

@@ -197,10 +197,13 @@
 		})(jQuery);
 		</script>
 		<script>
-			$.noConflict();
-			jQuery( document ).ready(function( $ ) {
-			 $('[data-toggle="popover"]').popover()
+		$.noConflict();
+		jQuery(function ($) {
+			$('[data-toggle="popover"]').popover();
+			$('#summary-table').on('all.bs.table', function (e, name, args) {
+				$('[data-toggle="popover"]').popover();
 			});
+		});
 		</script>
 		<link href="https://unpkg.com/bootstrap-table@1.15.4/dist/extensions/fixed-columns/bootstrap-table-fixed-columns.css" rel="stylesheet">
         <title>Looper: {{ project_name }} summary</title>
@@ -226,7 +229,7 @@ <h3 style="margin-bottom: 0px;">Looper <code>{{ project_name }}</code> summary</
         	</div>
 			<div class="row">
 				<div class="col-12">
-					<table data-toggle="table" data-search="true" data-page-size="10" data-pagination="true" data-show-columns="true" data-fixed-columns="true" data-sort-stable="true">
+					<table id="summary-table" data-toggle="table" data-search="true" data-page-size="50" data-pagination="true" data-show-columns="true" data-fixed-columns="true" data-sort-stable="true">
 						<thead class="thead-light">
 							<tr>
 								{% for column in columns %}
@@ -275,9 +278,7 @@ <h4 class="card-header">Plot a column</h4>
 					</div>
 				</div>
 			</div>
-			<div class="row">
-    			{{ project_objects }}
-			</div>
+			{{ project_objects }}
     	</div>
     </body>
     <hr class="featurette-divider">