Merge pull request #62 from FAST-HEP/BK_fix_sort_add_logging_control

Fix sort, add logging control

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.10.3] - 2021-07-02
+### Fixed
+- Postproc: Fix merge's sort option, add CLI logging controls, PR #62, [@DBAnthony](https://github.com/DBAnthony)
+
 ## [0.10.2] - 2021-06-24
 ### Fixed
 - Postproc: Fix default value for value_cols on command line, PR #60, [@DBAnthony](https://github.com/DBAnthony)

fast_plotter/postproc/__init__.py

@@ -1,4 +1,7 @@
+import logging
 from .functions import open_many
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
 
 
 __all__ = ["open_many"]

fast_plotter/postproc/__main__.py

@@ -1,16 +1,20 @@
 import logging
-from . import stages
+from . import stages, logger as base_logger
 from .functions import open_many
 from fast_flow.v1 import read_sequence_yaml
 from fast_flow.help import argparse_help_stages
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
+logger.setLevel(logging.NOTSET)
 
 
 def make_parser():
     from argparse import ArgumentParser
     parser = ArgumentParser()
-    parser.add_argument("-d", "--debug-dfs", default=False, action="store_true",
+    parser.add_argument('-q', '--quiet', default="INFO",
+                        action='store_const', const="WARNING", dest='verbosity',
+                        help="quiet output (show errors and warnings only)")
+    parser.add_argument("-d", "--debug-dfs",
+                        action="store_const", const="DEBUG", dest='verbosity',
                         help="Print a dataframe after each step")
     parser.add_argument("--debug-dfs-query", default=None,
                         help="Provide a query to select rows from the debugged dataframe")
@@ -43,7 +47,7 @@ def read_processing_cfg(fname, out_dir):
     return sequence
 
 
-def dump_debug_df(dfs, debug_dfs_query=""):
+def debug_df(dfs, debug_dfs_query=""):
     if not debug_dfs_query:
         return dfs[0][0]
 
@@ -54,26 +58,39 @@ def dump_debug_df(dfs, debug_dfs_query=""):
                 return debug_df
         except NameError:
             return None
-
-    logger.debug("No dataframes contain rows matching the debug-dfs-query")
     return None
 
 
+def dump_debug_df(dfs, debug_dfs_query="", debug_rows=5):
+    df = debug_df(dfs, debug_dfs_query)
+    if df is None:
+        logger.debug("No dataframes contain rows matching the debug-dfs-query")
+    else:
+        logger.debug(df.head(debug_rows).to_string())
+    return df
+
+
+def setup_logging(verbosity):
+    level = getattr(logging, verbosity)
+    base_logger.setLevel(level)
+    return verbosity == "DEBUG"
+
+
 def main(args=None):
     args = make_parser().parse_args(args=args)
-    if args.debug_dfs:
-        logger.setLevel(logging.DEBUG)
+    debug = setup_logging(args.verbosity)
 
     dfs = open_many(args.files, value_columns=args.value_cols)
+    if debug:
+        dump_debug_df(dfs, args.debug_dfs_query, args.debug_rows)
+
     sequence = read_processing_cfg(args.post_process, args.outdir)
 
     for stage in sequence:
         logger.info("Working on %d dataframes", len(dfs))
         dfs = stage(dfs)
-        if args.debug_dfs:
-            debug_df = dump_debug_df(dfs, args.debug_dfs_query)
-            if debug_df is not None:
-                logger.debug(debug_df.head(args.debug_rows).to_string())
+        if debug:
+            dump_debug_df(dfs, args.debug_dfs_query, args.debug_rows)
 
 
 if __name__ == "__main__":

fast_plotter/postproc/functions.py

@@ -6,7 +6,7 @@
 from .query_curator import prepare_datasets_scale_factor, make_dataset_map
 import logging
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
+logger.setLevel(logging.NOTSET)
 
 
 class BinningDimCombiner():
@@ -387,7 +387,7 @@ def merge(dfs, sort=True):
     """
     logger.info("Merging %d dataframes", len(dfs))
     final_df = pd.concat(dfs, sort=sort)  # .fillna(float("-inf"))
-    final_df = final_df.groupby(level=final_df.index.names).sum()  # .replace(float("-inf"), float("nan"))
+    final_df = final_df.groupby(level=final_df.index.names, sort=sort).sum()  # .replace(float("-inf"), float("nan"))
     return final_df
 
 

fast_plotter/version.py

@@ -12,5 +12,5 @@ def split_version(version):
     return tuple(result)
 
 
-__version__ = '0.10.2'
+__version__ = '0.10.3'
 version_info = split_version(__version__) # noqa

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.10.2
+current_version = 0.10.3
 commit = True
 tag = False