Merge pull request #23 from MyPyDavid/fix/make-example-run

Fix/make example run

.github/workflows/tox_actions.yml

@@ -27,7 +27,7 @@ jobs:
           - python-version: 3.9
             toxenv: "py39-cover,report" #,codecov"
         exclude:
-          # TODO py39 on windows has errors so we exclude it for now
+          # IDEA py39 on windows has errors so we exclude it for now
           - os: windows-latest
             python-version: 3.9
 

.gitignore

@@ -109,7 +109,7 @@ venv.bak/
 .mypy_cache/
 
 # pycharm
-.idea/
+.todo/
 
 # datafiles and results
 **/results/*

.pre-commit-config.yaml

@@ -49,7 +49,7 @@ repos:
         name: nbqa-black - consistent Python code formatting (notebooks; auto-fixes)
         args: [ --nbqa-mutate ]
         additional_dependencies: [ black==21.5b2 ]
-      # TODO: Disabled for now until it's clear how to add noqa to specific cells of a Jupyter notebook
+      # IDEA: Disabled for now until it's clear how to add noqa to specific cells of a Jupyter notebook
       #- id: nbqa-flake8
       #  name: nbqa-flake8 - Python linting (notebooks)
       #  additional_dependencies: [ flake8==3.9.2 ]

src/raman_fitting/__init__.py

@@ -23,7 +23,7 @@
 
 # VERSION_PATH = config.PACKAGE_ROOT / 'VERSION.txt'
 # with open(VERSION_PATH, 'r') as version_file:
-# TODO change version definitino
+# IDEA change version definitino
 # __version__ = version_file.read().strip()
 
 import logging
@@ -93,15 +93,3 @@
 # Other user-facing functions
 from .api import *
 
-# TODO list:
-# added setup.cfg
-# added unittests
-# added README.md
-# add project.toml only for
-# improved logger, each module needs a getlogger(name)
-# TODO future daemonize the fitting process for using the package and dropping files in the datafiles folder
-# TODO add docs with Sphinx, readthedocs
-# TODO improve AsyncIO into main delegator processes
-# TODO fix plotting because of DeprecationWarning in savefig
-# TODO add database for spectrum data storage
-# TODO future GUI webinterface

src/raman_fitting/config/filepath_helper.py

@@ -117,7 +117,7 @@ def create_dataset_dir(DATASET_DIR):  # pragma: no cover
 """
             )
             sys.exit()
-            # TODO build in daemon version with folder watcher....
+            # IDEA build in daemon version with folder watcher....
         except Exception as exc:
             logger.warning(
                 f"""The datafiles directory could not be created at:

src/raman_fitting/config/filepath_settings.py

@@ -60,7 +60,7 @@
 
 # ADAPT to your own configurations
 if 0:  # pragma: no cover
-    # TODO implement config file handling
+    # IDEA implement config file handling
     # LOCAL_CONFIG_FILE.is_file():
     try:
         # PACKAGE_ROOT, MODEL_DIR are not locally configurated

src/raman_fitting/deconvolution_models/base_model.py

@@ -111,7 +111,7 @@ class BaseModel:
     _SEP = "+"
     _SUFFIX = "_"
 
-    # TODO change include substrate to  has substrate and remove from init
+    # IDEA change include substrate to  has substrate and remove from init
     def __init__(
         self,
         model_name: str = "",
@@ -220,14 +220,14 @@ def add_substrate(self):
 
     def validate_model_name_input(self, value):
         """checks if given input name is valid"""
-        if not type(value) == str:
+        if type(value) != str:
             raise TypeError(
                 f'Given name "{value}" for model_name should be a string insteady of type({type(value).__name__})'
             )
         elif not value:
             warn(f'\n\tThis name "{value}" is an empty string', BaseModelWarning)
             return value
-        elif not "+" in value:
+        elif "+" not in value:
             warn(
                 f'\n\tThis name "{value}" does not contain the separator "+". (could be just 1 Peak)',
                 BaseModelWarning,

src/raman_fitting/deconvolution_models/default_peaks/first_order_peaks.py

@@ -139,5 +139,5 @@ def __init__(self):
 
 def test_for_Si_substrate(
     model,
-):  # TODO test fit on spectrum to decide wether Si substrate is required or not
+):  # IDEA test fit on spectrum to decide wether Si substrate is required or not
     """make test fit for only slice 900-1000"""

src/raman_fitting/deconvolution_models/fit_models.py

@@ -14,7 +14,7 @@ class Fitter:
     """
     Fitter class for executing the fitting functions and optimizations
 
-    # TODO implement sensing of spectrum for Si samples
+    IDEA: implement sensing of spectrum for Si samples
     """
 
     fit_windows = ["1st_order", "2nd_order"]
@@ -48,7 +48,7 @@ def spectra(self, value):
             _fit_lbl = "int"
         elif isinstance(value, pd.DataFrame):
             raise AttributeError
-            # TODO implement self.sense_windowname(value)
+            # IDEA implement self.sense_windowname(value)
         else:
             raise ValueError(_errtxt)
 
@@ -112,7 +112,7 @@ def fit_models(self, model_selection):
         self.FitResults.update(**_fittings)
 
     def run_fit(self, model, _data, method="leastsq", **kws):
-        # TODO improve fitting loop so that starting parameters from modelX and modelX+Si are shared, faster...
+        # ideas: improve fitting loop so that starting parameters from modelX and modelX+Si are shared, faster...
         _fit_res, _param_res = {}, {}
         init_params = model.make_params()
         x, y = _data.ramanshift, _data[kws.get("_int_lbl")]
@@ -131,7 +131,7 @@ def run_fit(self, model, _data, method="leastsq", **kws):
     def get_int_label(self, value):
         _lbl = ""
         if isinstance(value, pd.DataFrame):
-            cols = [i for i in value.columns if not "ramanshift" in i]
+            cols = [i for i in value.columns if "ramanshift" not in i]
             if len(cols) == 0:
                 _lbl = ""
             if len(cols) == 1:
@@ -206,15 +206,10 @@ def model_result(self, value):
             _mod_lbl = f'Model_{getattr(value,"_modelname")}'
         self.model_name_lbl = _mod_lbl
 
-        # TODO remove
-        # from pprint import pprint
-        # print('=====  /n',value._int_lbl,'/n')
-
         self.raw_data_lbl = value._int_lbl
 
         self._model_result = value
 
-        # TODO rewrite class and attirbuter setter
         self.make_result()
 
     def make_result(self):
@@ -335,7 +330,7 @@ def prep_components(self):
         _fit_comps_data = OrderedDict({"RamanShift": self.model_result.userkws["x"]})
         _fit_comps_data.update(self.model_result.eval_components())
 
-        # TODO take out
+        # IDEA take out
         # print('===/n',self.model_result, '/n')
         # print('===/n',self.model_result.__dict__.keys(), '/n')
 
@@ -344,7 +339,6 @@ def prep_components(self):
             {
                 self.model_name_lbl: self.model_result.best_fit,
                 "residuals": self.model_result.residual,
-                # TODO check attributes of model_result
                 self.model_result._int_lbl: self.model_result.data,
             }
         )
@@ -353,7 +347,7 @@ def prep_components(self):
 
 
 def NormalizeFit(norm_cleaner, plotprint=False):  # pragma: no cover
-    # TODO optional add normalization seperately to Fitter
+    # IDEA: optional add normalization seperately to Fitter
     x, y = norm_cleaner.spec.ramanshift, norm_cleaner.blcorr_desp_intensity
     Model = InitializeModels("2peaks normalization Lorentzian")
     params = Model.make_params()

src/raman_fitting/deconvolution_models/peak_validation.py

@@ -342,7 +342,7 @@ def __getattr__(self, name):
         # raise AttributeError(f'Chosen name "{name}" not in in options: "{", ".join(self.options)}".')
 
     def normalization_model(self):
-        pass  # TODO separate peaks in groups
+        pass  # IDEA separate peaks in groups
 
     def __iter__(self):
         for mod_inst in self.lmfit_models:

src/raman_fitting/delegating/main_delegator.py

@@ -57,7 +57,7 @@ class MainDelegatorError(prdError):
 
 
 class MainDelegator:
-    # TODO Add flexible input handling for the cli, such a path to dir, or list of files
+    # IDEA Add flexible input handling for the cli, such a path to dir, or list of files
     #  or create index when no kwargs are given.
     """
     Main delegator for the processing of files containing Raman spectra.
@@ -104,7 +104,7 @@ def index_delegator(self, **kwargs):
         return RF_index
 
     def run_delegator(self, **kwargs):
-        # TODO remove self.set_models() removed InitModels
+        # IDEA remove self.set_models() removed InitModels
         self._failed_samples = []
         self.export_collect = []
 
@@ -126,7 +126,7 @@ def run_delegator(self, **kwargs):
 
             models = self.initialize_default_models()
             self.kwargs.update({"models": models})
-            # TODO built in a model selection keyword, here or at fitting level and for the cli
+            # IDEA built in a model selection keyword, here or at fitting level and for the cli
             logger.info(
                 f"\n{self._cqnm} models initialized for run mode ({self.run_mode}):\n\n{repr(models)}"
             )
@@ -144,7 +144,7 @@ def run_delegator(self, **kwargs):
                 logger.debug(f"Debug run mode {self}. Models initialized {models}")
 
                 try:
-                    # self._run_gen() # TODO add extra test runs in tests dir
+                    # self._run_gen() # IDEA add extra test runs in tests dir
                     pass
                 except Exception as e:
                     raise MainDelegatorError(
@@ -153,7 +153,7 @@ def run_delegator(self, **kwargs):
                 # raise('Error in DEBUG run: ', e)
             else:
                 logger.warning(f"Debug run mode {self.run_mode} not recognized")
-            # TODO get testing from index and run
+            # IDEA get testing from index and run
         else:
             logger.warning(f'Debug run mode "{self.run_mode}" not recognized not in ')
             # warning run mode not recognized
@@ -182,11 +182,14 @@ def _sID_gen(self, grpnm, sGrp_grp):
             yield (grpnm, nm, sID_grp)
 
     def _run_gen(self, **kwargs):
-        # #TODO sort of coordinator coroutine, can implement still deque
+        # #IDEA sort of coordinator coroutine, can implement still deque
         sgrp_grpby = self.index.groupby(self.spectrum.grp_names.sGrp_cols[0])
         logger.info(f"{self._cqnm} _run_gen starting: {kwargs}")
         _mygen = self._generator(sgrp_grpby, **kwargs)
-        Exporter(self.export_collect)  # clean up and export
+        Exporter(self.export_collect)  # clean up and 
+        logger.info(
+                    f"\n{self._cqnm} run finished.\n Results saved in {self.RESULTS_DIR}"
+                )
 
     def _generator(self, sgrp_grpby, **kwargs):
         # _sgrpgen = self.sample_group_gen()
@@ -244,20 +247,6 @@ def simple_process_sample_wrapper(self, *sID_args, **kwargs):
             self._failed_samples.append((e, sID_args, kwargs))
         return exporter_sample
 
-    def _process_sample_wrapper(self, fn, *args, **kwargs):
-        logger.warning(
-            f"{self._cqnm} process_sample_wrapper args:\n\t - {fn}\n\t - {args}\n\t - {kwargs.keys()}"
-        )
-        exp_sample = None
-        try:
-            exp_sample = fn(self, *args, **kwargs)
-            self.export_collect.append(exp_sample)
-        except Exception as e:
-            logger.warning(
-                f"{self._cqnm} process_sample_wrapper exception on call {fn}: {e}"
-            )
-            self._failed_samples.append((e, args, kwargs))
-
     def test_positions(
         self, sGrp_grp, sIDnm, grp_cols=["FileStem", "SamplePos", "FilePath"]
     ):

src/raman_fitting/exporting/exporter.py

@@ -135,7 +135,7 @@ def export_xls_from_spec(self, res_peak_spec):
             # res_peak_spec.extrainfo.to_excel(peak_destpath_extra.with_suffix('.xlsx'), index=False)
         except Exception as e:
             print("Error export_xls_from_spec", e)
-        # TODO define fuction for exporting all the indexes _all_index_export
+        # IDEA define fuction for exporting all the indexes _all_index_export
         # index = RamanExport().export_FitParams_Grp(FitParams1, FitParams2, export_info_out, grpnm,sID)
         # all_index.append(index)
         # pars_index = pd.DataFrame(*all_index,columns=list(GrpNames.sGrp_cols[0:2] +('PeakModel','DestPars')))

src/raman_fitting/exporting/plotting.py

@@ -15,7 +15,7 @@
 
 #%%
 
-# TODO FIX PLOTTING PER PEAK MODEL
+# IDEA PLOTTING PER PEAK MODEL
 def plotting_info(windowname):  # pragma: no cover
     axes = {
         "full": (0, 0),
@@ -117,6 +117,7 @@ def fit_spectrum_plot(
     plot_Residuals=True,
 ):  # pragma: no cover
 
+    modname_2 = peak2
     #%%
     sID = res1_peak_spec.extrainfo["SampleID"]
     SampleBgmean_col = res1_peak_spec.raw_data_col
@@ -178,14 +179,15 @@ def fit_spectrum_plot(
         c="grey",
         alpha=0.5,
     )
-    ax2ndRes.plot(
-        FitData_2nd["RamanShift"],
-        FitData_2nd[res2_peak_spec.raw_data_col] - FitData_2nd[Model_data_col_2nd],
-        label="Residual",
-        lw=3,
-        c="k",
-        alpha=0.8,
-    )
+    if plot_Residuals:
+        ax2ndRes.plot(
+            FitData_2nd["RamanShift"],
+            FitData_2nd[res2_peak_spec.raw_data_col] - FitData_2nd[Model_data_col_2nd],
+            label="Residual",
+            lw=3,
+            c="k",
+            alpha=0.8,
+        )
 
     for fit_comp_col_2nd in compscols_2nd:  # automatic color cycle 'cyan' ...
         ax2nd.plot(
@@ -227,14 +229,16 @@ def fit_spectrum_plot(
         c="grey",
         alpha=0.8,
     )
-    axRes.plot(
-        FitData_1st["RamanShift"],
-        FitData_1st[res1_peak_spec.raw_data_col] - FitData_1st[Model_data_col_1st],
-        label="Residual",
-        lw=3,
-        c="k",
-        alpha=0.8,
-    )
+
+    if plot_Residuals:
+        axRes.plot(
+            FitData_1st["RamanShift"],
+            FitData_1st[res1_peak_spec.raw_data_col] - FitData_1st[Model_data_col_1st],
+            label="Residual",
+            lw=3,
+            c="k",
+            alpha=0.8,
+        )
 
     for fit_comp_col_1st in compscols_1st:  # automatic color cycle 'cyan' ...
         ax.plot(
@@ -324,7 +328,7 @@ def fit_spectrum_plot(
     plt.savefig(
         res1_peak_spec.extrainfo["DestFittingModel"].with_suffix(".png"),
         dpi=100,
-        box_extra_artists=(Report1, Report2),
+        bbox_extra_artists=(Report1, Report2),
         bbox_inches="tight",
     )
     plt.close()