diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ad789d..15f11b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.10.3] - 2021-07-02 +### Fixed +- Postproc: Fix merge's sort option, add CLI logging controls, PR #62, [@DBAnthony](https://github.com/DBAnthony) + ## [0.10.2] - 2021-06-24 ### Fixed - Postproc: Fix default value for value_cols on command line, PR #60, [@DBAnthony](https://github.com/DBAnthony) diff --git a/fast_plotter/postproc/__init__.py b/fast_plotter/postproc/__init__.py index 052d6ed..e28f24b 100644 --- a/fast_plotter/postproc/__init__.py +++ b/fast_plotter/postproc/__init__.py @@ -1,4 +1,7 @@ +import logging from .functions import open_many +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) __all__ = ["open_many"] diff --git a/fast_plotter/postproc/__main__.py b/fast_plotter/postproc/__main__.py index 631ce92..2a513f6 100644 --- a/fast_plotter/postproc/__main__.py +++ b/fast_plotter/postproc/__main__.py @@ -1,16 +1,20 @@ import logging -from . import stages +from . import stages, logger as base_logger from .functions import open_many from fast_flow.v1 import read_sequence_yaml from fast_flow.help import argparse_help_stages logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) +logger.setLevel(logging.NOTSET) def make_parser(): from argparse import ArgumentParser parser = ArgumentParser() - parser.add_argument("-d", "--debug-dfs", default=False, action="store_true", + parser.add_argument('-q', '--quiet', default="INFO", + action='store_const', const="WARNING", dest='verbosity', + help="quiet output (show errors and warnings only)") + parser.add_argument("-d", "--debug-dfs", + action="store_const", const="DEBUG", dest='verbosity', help="Print a dataframe after each step") parser.add_argument("--debug-dfs-query", default=None, help="Provide a query to select rows from the debugged dataframe") @@ -43,7 +47,7 @@ def read_processing_cfg(fname, out_dir): return sequence -def dump_debug_df(dfs, debug_dfs_query=""): +def debug_df(dfs, debug_dfs_query=""): if not debug_dfs_query: return dfs[0][0] @@ -54,26 +58,39 @@ def dump_debug_df(dfs, debug_dfs_query=""): return debug_df except NameError: return None - - logger.debug("No dataframes contain rows matching the debug-dfs-query") return None +def dump_debug_df(dfs, debug_dfs_query="", debug_rows=5): + df = debug_df(dfs, debug_dfs_query) + if df is None: + logger.debug("No dataframes contain rows matching the debug-dfs-query") + else: + logger.debug(df.head(debug_rows).to_string()) + return df + + +def setup_logging(verbosity): + level = getattr(logging, verbosity) + base_logger.setLevel(level) + return verbosity == "DEBUG" + + def main(args=None): args = make_parser().parse_args(args=args) - if args.debug_dfs: - logger.setLevel(logging.DEBUG) + debug = setup_logging(args.verbosity) dfs = open_many(args.files, value_columns=args.value_cols) + if debug: + dump_debug_df(dfs, args.debug_dfs_query, args.debug_rows) + sequence = read_processing_cfg(args.post_process, args.outdir) for stage in sequence: logger.info("Working on %d dataframes", len(dfs)) dfs = stage(dfs) - if args.debug_dfs: - debug_df = dump_debug_df(dfs, args.debug_dfs_query) - if debug_df is not None: - logger.debug(debug_df.head(args.debug_rows).to_string()) + if debug: + dump_debug_df(dfs, args.debug_dfs_query, args.debug_rows) if __name__ == "__main__": diff --git a/fast_plotter/postproc/functions.py b/fast_plotter/postproc/functions.py index 8d6e5ba..8c0ffb0 100644 --- a/fast_plotter/postproc/functions.py +++ b/fast_plotter/postproc/functions.py @@ -6,7 +6,7 @@ from .query_curator import prepare_datasets_scale_factor, make_dataset_map import logging logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) +logger.setLevel(logging.NOTSET) class BinningDimCombiner(): @@ -387,7 +387,7 @@ def merge(dfs, sort=True): """ logger.info("Merging %d dataframes", len(dfs)) final_df = pd.concat(dfs, sort=sort) # .fillna(float("-inf")) - final_df = final_df.groupby(level=final_df.index.names).sum() # .replace(float("-inf"), float("nan")) + final_df = final_df.groupby(level=final_df.index.names, sort=sort).sum() # .replace(float("-inf"), float("nan")) return final_df diff --git a/fast_plotter/version.py b/fast_plotter/version.py index 0a048c3..263b7e9 100644 --- a/fast_plotter/version.py +++ b/fast_plotter/version.py @@ -12,5 +12,5 @@ def split_version(version): return tuple(result) -__version__ = '0.10.2' +__version__ = '0.10.3' version_info = split_version(__version__) # noqa diff --git a/setup.cfg b/setup.cfg index e51cdf4..ae0357b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.10.2 +current_version = 0.10.3 commit = True tag = False