From fda809727e2d37fd9e110d2f3c707e4bc58062e1 Mon Sep 17 00:00:00 2001 From: Ben Krikler Date: Thu, 1 Jul 2021 12:27:41 +0200 Subject: [PATCH 1/5] Fix sort issue when merging --- fast_plotter/postproc/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast_plotter/postproc/functions.py b/fast_plotter/postproc/functions.py index 8d6e5ba..4e5fdce 100644 --- a/fast_plotter/postproc/functions.py +++ b/fast_plotter/postproc/functions.py @@ -387,7 +387,7 @@ def merge(dfs, sort=True): """ logger.info("Merging %d dataframes", len(dfs)) final_df = pd.concat(dfs, sort=sort) # .fillna(float("-inf")) - final_df = final_df.groupby(level=final_df.index.names).sum() # .replace(float("-inf"), float("nan")) + final_df = final_df.groupby(level=final_df.index.names, sort=sort).sum() # .replace(float("-inf"), float("nan")) return final_df From 3a862232445aae85c7b0e74753d1079d8e8bad5c Mon Sep 17 00:00:00 2001 From: Ben Krikler Date: Thu, 1 Jul 2021 12:30:07 +0200 Subject: [PATCH 2/5] Refactor debugging output --- fast_plotter/postproc/__main__.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/fast_plotter/postproc/__main__.py b/fast_plotter/postproc/__main__.py index 631ce92..acbb443 100644 --- a/fast_plotter/postproc/__main__.py +++ b/fast_plotter/postproc/__main__.py @@ -43,7 +43,7 @@ def read_processing_cfg(fname, out_dir): return sequence -def dump_debug_df(dfs, debug_dfs_query=""): +def debug_df(dfs, debug_dfs_query=""): if not debug_dfs_query: return dfs[0][0] @@ -54,26 +54,34 @@ def dump_debug_df(dfs, debug_dfs_query=""): return debug_df except NameError: return None - - logger.debug("No dataframes contain rows matching the debug-dfs-query") return None +def dump_debug_df(dfs, debug_dfs_query="", debug_rows=5): + df = debug_df(dfs, debug_dfs_query) + if df is None: + logging.debug("No dataframes contain rows matching the debug-dfs-query") + else: + logging.debug(df.head(debug_rows).to_string()) + return df + + def main(args=None): args = make_parser().parse_args(args=args) if args.debug_dfs: logger.setLevel(logging.DEBUG) dfs = open_many(args.files, value_columns=args.value_cols) + if args.debug_dfs: + dump_debug_df(dfs, args.debug_dfs_query, args.debug_rows) + sequence = read_processing_cfg(args.post_process, args.outdir) for stage in sequence: logger.info("Working on %d dataframes", len(dfs)) dfs = stage(dfs) if args.debug_dfs: - debug_df = dump_debug_df(dfs, args.debug_dfs_query) - if debug_df is not None: - logger.debug(debug_df.head(args.debug_rows).to_string()) + dump_debug_df(dfs, args.debug_dfs_query, args.debug_rows) if __name__ == "__main__": From 9e682fd3dcf4715ccd3f11b89ae27d01973f585d Mon Sep 17 00:00:00 2001 From: Ben Krikler Date: Fri, 2 Jul 2021 06:05:14 +0200 Subject: [PATCH 3/5] Sort out logging options --- fast_plotter/postproc/__init__.py | 3 +++ fast_plotter/postproc/__main__.py | 27 ++++++++++++++++++--------- fast_plotter/postproc/functions.py | 2 +- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/fast_plotter/postproc/__init__.py b/fast_plotter/postproc/__init__.py index 052d6ed..e28f24b 100644 --- a/fast_plotter/postproc/__init__.py +++ b/fast_plotter/postproc/__init__.py @@ -1,4 +1,7 @@ +import logging from .functions import open_many +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) __all__ = ["open_many"] diff --git a/fast_plotter/postproc/__main__.py b/fast_plotter/postproc/__main__.py index acbb443..2a513f6 100644 --- a/fast_plotter/postproc/__main__.py +++ b/fast_plotter/postproc/__main__.py @@ -1,16 +1,20 @@ import logging -from . import stages +from . import stages, logger as base_logger from .functions import open_many from fast_flow.v1 import read_sequence_yaml from fast_flow.help import argparse_help_stages logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) +logger.setLevel(logging.NOTSET) def make_parser(): from argparse import ArgumentParser parser = ArgumentParser() - parser.add_argument("-d", "--debug-dfs", default=False, action="store_true", + parser.add_argument('-q', '--quiet', default="INFO", + action='store_const', const="WARNING", dest='verbosity', + help="quiet output (show errors and warnings only)") + parser.add_argument("-d", "--debug-dfs", + action="store_const", const="DEBUG", dest='verbosity', help="Print a dataframe after each step") parser.add_argument("--debug-dfs-query", default=None, help="Provide a query to select rows from the debugged dataframe") @@ -60,19 +64,24 @@ def debug_df(dfs, debug_dfs_query=""): def dump_debug_df(dfs, debug_dfs_query="", debug_rows=5): df = debug_df(dfs, debug_dfs_query) if df is None: - logging.debug("No dataframes contain rows matching the debug-dfs-query") + logger.debug("No dataframes contain rows matching the debug-dfs-query") else: - logging.debug(df.head(debug_rows).to_string()) + logger.debug(df.head(debug_rows).to_string()) return df +def setup_logging(verbosity): + level = getattr(logging, verbosity) + base_logger.setLevel(level) + return verbosity == "DEBUG" + + def main(args=None): args = make_parser().parse_args(args=args) - if args.debug_dfs: - logger.setLevel(logging.DEBUG) + debug = setup_logging(args.verbosity) dfs = open_many(args.files, value_columns=args.value_cols) - if args.debug_dfs: + if debug: dump_debug_df(dfs, args.debug_dfs_query, args.debug_rows) sequence = read_processing_cfg(args.post_process, args.outdir) @@ -80,7 +89,7 @@ def main(args=None): for stage in sequence: logger.info("Working on %d dataframes", len(dfs)) dfs = stage(dfs) - if args.debug_dfs: + if debug: dump_debug_df(dfs, args.debug_dfs_query, args.debug_rows) diff --git a/fast_plotter/postproc/functions.py b/fast_plotter/postproc/functions.py index 4e5fdce..8c0ffb0 100644 --- a/fast_plotter/postproc/functions.py +++ b/fast_plotter/postproc/functions.py @@ -6,7 +6,7 @@ from .query_curator import prepare_datasets_scale_factor, make_dataset_map import logging logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) +logger.setLevel(logging.NOTSET) class BinningDimCombiner(): From 6d08a74e67a91b5c0744cf8e28af99b28af29127 Mon Sep 17 00:00:00 2001 From: Ben Krikler Date: Fri, 2 Jul 2021 06:05:43 +0200 Subject: [PATCH 4/5] =?UTF-8?q?Bump=20version:=200.10.2=20=E2=86=92=200.10?= =?UTF-8?q?.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fast_plotter/version.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fast_plotter/version.py b/fast_plotter/version.py index 0a048c3..263b7e9 100644 --- a/fast_plotter/version.py +++ b/fast_plotter/version.py @@ -12,5 +12,5 @@ def split_version(version): return tuple(result) -__version__ = '0.10.2' +__version__ = '0.10.3' version_info = split_version(__version__) # noqa diff --git a/setup.cfg b/setup.cfg index e51cdf4..ae0357b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.10.2 +current_version = 0.10.3 commit = True tag = False From 2100ac63fa337ed7a75531331a01184edf0f098c Mon Sep 17 00:00:00 2001 From: Ben Krikler Date: Fri, 2 Jul 2021 06:09:38 +0200 Subject: [PATCH 5/5] Update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ad789d..15f11b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.10.3] - 2021-07-02 +### Fixed +- Postproc: Fix merge's sort option, add CLI logging controls, PR #62, [@DBAnthony](https://github.com/DBAnthony) + ## [0.10.2] - 2021-06-24 ### Fixed - Postproc: Fix default value for value_cols on command line, PR #60, [@DBAnthony](https://github.com/DBAnthony)