Refactor loading data (#305)

* refactor loading data

* fix batch processing test

* cleanup and removing old code

* remove deprecated import

* cleanup imports

* fix gui tests

* template imports

* allow filtering exposure by float

* fix typo

* fix column order

* update examples

* types

* add global metadata

* ran black

dev/deps/_requirements.py

@@ -23,7 +23,6 @@
 from typing import Optional
 
 
-
 #%%
 # Pycharm scientific mode compat
 if "__file__" not in locals():
@@ -144,5 +143,4 @@ def make_requirements_files(extras: Optional[list[str]] = None):
         conda_file = Path(f"pinned/py38_{os}_conda.yml")
         pip_file = Path(f"pinned/py38_{os}_pip.txt")
 
-
         conda_to_pip(conda_file, pip_file)

dev/gui/dev_gui_secB.py

@@ -5,7 +5,6 @@
 """
 
 
-
 import sys
 from pathlib import Path
 
@@ -21,10 +20,10 @@
 from pyhdx.web.utils import load_state, fix_multiindex_dtypes
 from pyhdx.config import cfg, reset_config
 
-#def printfunc(args):
-    # 1/0
+# def printfunc(args):
+# 1/0
 
-#sys.stdout = printfunc
+# sys.stdout = printfunc
 
 
 sys._excepthook = sys.excepthook
@@ -34,6 +33,8 @@
 
 
 import traceback as tb
+
+
 def my_exception_hook(exctype, value, traceback):
     # Print the error and traceback
     # https://stackoverflow.com/questions/43039048/pyqt5-fails-with-cryptic-message/43039363#43039363
@@ -48,6 +49,7 @@ def my_exception_hook(exctype, value, traceback):
     sys._excepthook(exctype, value, traceback)
     sys.exit(1)
 
+
 # Set the exception hook to our wrapping function
 sys.excepthook = my_exception_hook
 
@@ -57,57 +59,71 @@ def my_exception_hook(exctype, value, traceback):
 cwd = Path(__file__).parent
 root_dir = cwd.parent.parent
 
-web_data_dir = root_dir / 'tests' / 'test_data' / 'output' / 'web'
-input_data_dir = root_dir / 'tests' / 'test_data' / 'input'
+web_data_dir = root_dir / "tests" / "test_data" / "output" / "web"
+input_data_dir = root_dir / "tests" / "test_data" / "input"
 
 
-filenames = ['ecSecB_apo.csv', 'ecSecB_dimer.csv']
+filenames = ["ecSecB_apo.csv", "ecSecB_dimer.csv"]
 file_dict = {fname: (input_data_dir / fname).read_bytes() for fname in filenames}
 
-batch_fname = 'data_states_deltas.yaml' # secb apo / dimer but artificial delta C/N tail
+# batch_fname = 'data_states_deltas.yaml' # secb apo / dimer but artificial delta C/N tail, needs additional files
+batch_fname = "data_states.yaml"  #
+
 
 state_spec = yaml.safe_load(Path(input_data_dir / batch_fname).read_text())
 # pdb_string = (web_data_dir / '1qyn.pdb').read_text()
-pdb_string = (web_data_dir / '5JTR_mod.pdb').read_text()
+pdb_string = (web_data_dir / "5JTR_mod.pdb").read_text()
 
 
 def reload_tables():
-    src = ctrl.sources['main']
-    src.add_table('peptides', csv_to_dataframe(web_data_dir / 'peptides.csv'))
-    src.add_table('d_uptake', csv_to_dataframe(web_data_dir / 'd_uptake.csv'))
-    src.add_table('rfu', csv_to_dataframe(web_data_dir / 'rfu.csv'))
-    src.add_table('dG', csv_to_dataframe(web_data_dir / 'dG.csv'))
-    src.add_table('ddG_comparison', csv_to_dataframe(web_data_dir / 'ddG_comparison.csv'))
-    src.add_table('rates', csv_to_dataframe(web_data_dir / 'rates.csv'))
-    src.param.trigger('updated')
+    src = ctrl.sources["main"]
+    src.add_table("peptides", csv_to_dataframe(web_data_dir / "peptides.csv"))
+    src.add_table("d_uptake", csv_to_dataframe(web_data_dir / "d_uptake.csv"))
+    src.add_table("rfu", csv_to_dataframe(web_data_dir / "rfu.csv"))
+    src.add_table("dG", csv_to_dataframe(web_data_dir / "dG.csv"))
+    src.add_table(
+        "ddG_comparison", csv_to_dataframe(web_data_dir / "ddG_comparison.csv")
+    )
+    src.add_table("rates", csv_to_dataframe(web_data_dir / "rates.csv"))
+    src.param.trigger("updated")
 
     # ctrl.views['protein'].object = pdb_string
 
+
 def reload_dashboard():
-    source = ctrl.sources['dataframe']
-    for ds in ['peptides', 'peptides_mse', 'd_calc', 'rfu', 'rates', 'global_fit', 'losses']:
-        df = csv_to_dataframe(web_data_dir / f'{ds}.csv')
+    source = ctrl.sources["dataframe"]
+    for ds in [
+        "peptides",
+        "peptides_mse",
+        "d_calc",
+        "rfu",
+        "rates",
+        "global_fit",
+        "losses",
+    ]:
+        df = csv_to_dataframe(web_data_dir / f"{ds}.csv")
         source.add_df(df, ds)
 
-    #Temporary workaround for comment characters in csv files
-    ds = 'colors'
-    df = pd.read_csv(web_data_dir / f'{ds}.csv', header=[0, 1, 2], index_col=0,
-                     skiprows=3)
+    # Temporary workaround for comment characters in csv files
+    ds = "colors"
+    df = pd.read_csv(
+        web_data_dir / f"{ds}.csv", header=[0, 1, 2], index_col=0, skiprows=3
+    )
     source.add_df(df, ds)
 
+
 def init_batch():
-    input_control = ctrl.control_panels['PeptideFileInputControl']
-    input_control.input_mode = 'Batch'
+    input_control = ctrl.control_panels["PeptideFileInputControl"]
+    input_control.input_mode = "Batch"
+
+    file_dict = {fname: (input_data_dir / fname).read_bytes() for fname in filenames}
+    input_control.widgets["input_files"].filename = list(file_dict.keys())
     input_control.input_files = list(file_dict.values())
-    input_control.widgets['input_files'].filename = list(file_dict.keys())
 
     input_control.batch_file = Path(input_data_dir / batch_fname).read_bytes()
     input_control._action_load_datasets()
 
-    input_control.batch_file = Path(input_data_dir / batch_fname).read_bytes()
-    input_control._action_add_dataset()
-
-    fit_control = ctrl.control_panels['FitControl']
+    fit_control = ctrl.control_panels["FitControl"]
 
     fit_control.r1 = 0.05
     fit_control.r2 = 0.1
@@ -119,7 +135,7 @@ def init_batch():
 
 def init_dashboard():
     n = 2  # change this to control the number of HDX measurements added
-    input_control = ctrl.control_panels['PeptideFileInputControl']
+    input_control = ctrl.control_panels["PeptideFileInputControl"]
 
     for i, (k, v) in enumerate(state_spec.items()):
         if i == n:
@@ -132,9 +148,9 @@ def init_dashboard():
     d_uptake_control = ctrl.control_panels["DUptakeFitControl"]
     d_uptake_control.repeats = 2
 
-    ctrl.sources['pdb'].add_from_string(pdb_string, '1qyn')
+    ctrl.sources["pdb"].add_from_string(pdb_string, "1qyn")
 
-    src = ctrl.sources['main']
+    src = ctrl.sources["main"]
     # df = csv_to_dataframe(web_data_dir / 'd_uptake.csv')
     # df.columns = fix_multiindex_dtypes(df.columns)
     # src.add_table('d_uptake', df)
@@ -146,7 +162,6 @@ def init_dashboard():
 
     # src.updated = True
 
-
     # guess_control = ctrl.control_panels['InitialGuessControl']
     # guess_control._action_fit()
     #
@@ -170,23 +185,39 @@ def init_dashboard():
     #     diff = ctrl.control_panels['DifferentialControl']
     #     diff._action_add_comparison()
 
-
     # if n > 1:
     #     diff = ctrl.control_panels['DifferentialControl']
     #     diff._action_add_comparison()
 
 
-#pn.state.onload(reload_dashboard)
-#pn.state.onload(reload_tables)
-pn.state.onload(init_dashboard)
-#pn.state.onload(init_batch)
+def init_manual():
+    input_control = ctrl.control_panels["PeptideFileInputControl"]
+
+    file_dict = {fname: (input_data_dir / fname).read_bytes() for fname in filenames}
+    input_control.widgets["input_files"].filename = list(file_dict.keys())
+    input_control.input_files = list(file_dict.values())
+
+    input_control.fd_state = "Full deuteration control"
+    input_control.fd_exposure = 10.020000000000001
+
+    input_control.exp_state = "SecB WT apo"
+
 
+# pn.state.onload(reload_dashboard)
+# pn.state.onload(reload_tables)
+# pn.state.onload(init_dashboard)
+pn.state.onload(init_batch)
+# pn.state.onload(init_manual)
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     Path(cfg.assets_dir).mkdir(exist_ok=True, parents=True)
-    pn.serve(tmpl, show=True, static_dirs={'pyhdx': STATIC_DIR, "assets": str(cfg.assets_dir)})
+    pn.serve(
+        tmpl,
+        show=True,
+        static_dirs={"pyhdx": STATIC_DIR, "assets": str(cfg.assets_dir)},
+    )
 
-elif __name__.startswith('bokeh_app'):
+elif __name__.startswith("bokeh_app"):
     Path(cfg.assets_dir).mkdir(exist_ok=True, parents=True)
     tmpl.servable()
 

dev/gui/dev_mwe_app.py

@@ -56,12 +56,7 @@ def my_exception_hook(exctype, value, traceback):
 views = {v: ctrl.views[v] for v in views_names}
 # [v.update() for v in views.values()]
 
-tmpl = elvis.compose(
-    elvis.row(
-        elvis.view("xy_scatter"),
-        elvis.view("xy_line")
-    )
-)
+tmpl = elvis.compose(elvis.row(elvis.view("xy_scatter"), elvis.view("xy_line")))
 
 
 def reload_tables():

dev/gui/dev_rfu_app.py

@@ -23,6 +23,8 @@
 sys._excepthook = sys.excepthook
 
 import traceback as tb
+
+
 def my_exception_hook(exctype, value, traceback):
     # Print the error and traceback
     # https://stackoverflow.com/questions/43039048/pyqt5-fails-with-cryptic-message/43039363#43039363
@@ -37,6 +39,7 @@ def my_exception_hook(exctype, value, traceback):
     sys._excepthook(exctype, value, traceback)
     sys.exit(1)
 
+
 # Set the exception hook to our wrapping function
 sys.excepthook = my_exception_hook
 
@@ -45,19 +48,19 @@ def my_exception_hook(exctype, value, traceback):
 
 cwd = Path(__file__).parent
 root_dir = cwd.parent.parent
-data_dir = root_dir / 'tests' / 'test_data' / 'input'
+data_dir = root_dir / "tests" / "test_data" / "input"
 
 
 # batch_fname = 'data_states.yaml' # standard secb apo / dimer dataset
 # batch_fname = 'data_states_red.yaml'  # reduced number of pepties
-batch_fname = 'PpiX_states.yaml' # secb apo / dimer but artificial delta C/N tail
+batch_fname = "PpiX_states.yaml"  # secb apo / dimer but artificial delta C/N tail
 
 state_spec = yaml.safe_load(Path(data_dir / batch_fname).read_text())
 
 
 def init_dashboard():
     n = 2  # change this to control the number of HDX measurements added
-    input_control = ctrl.control_panels['PeptideRFUFileInputControl']
+    input_control = ctrl.control_panels["PeptideRFUFileInputControl"]
     for i, (k, v) in enumerate(state_spec.items()):
         if i == n:
             break
@@ -66,23 +69,26 @@ def init_dashboard():
 
     input_control._action_load_datasets()
 
-
-
     # if n > 1:
     #     diff = ctrl.control_panels['DifferentialControl']
     #     diff._action_add_comparison()
 
-#pn.state.onload(reload_dashboard)
-#pn.state.onload(reload_tables)
+
+# pn.state.onload(reload_dashboard)
+# pn.state.onload(reload_tables)
 pn.state.onload(init_dashboard)
-#pn.state.onload(init_batch)
+# pn.state.onload(init_batch)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     Path(cfg.assets_dir).mkdir(exist_ok=True, parents=True)
-    pn.serve(tmpl, show=True, static_dirs={'pyhdx': STATIC_DIR, "assets": str(cfg.assets_dir)})
+    pn.serve(
+        tmpl,
+        show=True,
+        static_dirs={"pyhdx": STATIC_DIR, "assets": str(cfg.assets_dir)},
+    )
 
-elif __name__.startswith('bokeh_app'):
+elif __name__.startswith("bokeh_app"):
     Path(cfg.assets_dir).mkdir(exist_ok=True, parents=True)
     tmpl.servable()
 

docs/bk/conf.py

@@ -72,9 +72,9 @@ def setup(app):
 master_doc = "index"
 
 # General information about the project.
-project = u"PyHDX"
-copyright = u"2022, Jochem Smit"
-author = u"Jochem Smit"
+project = "PyHDX"
+copyright = "2022, Jochem Smit"
+author = "Jochem Smit"
 
 # The version info for the project you're documenting, acts as replacement
 # for |version| and |release|, also used in various other places throughout
@@ -150,15 +150,15 @@ def setup(app):
 # (source start file, target name, title, author, documentclass
 # [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, "pyhdx.tex", u"PyHDX Documentation", u"Jochem Smit", "manual"),
+    (master_doc, "pyhdx.tex", "PyHDX Documentation", "Jochem Smit", "manual"),
 ]
 
 
 # -- Options for manual page output ------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [(master_doc, "pyhdx", u"PyHDX Documentation", [author], 1)]
+man_pages = [(master_doc, "pyhdx", "PyHDX Documentation", [author], 1)]
 
 
 # -- Options for Texinfo output ----------------------------------------
@@ -170,7 +170,7 @@ def setup(app):
     (
         master_doc,
         "pyhdx",
-        u"PyHDX Documentation",
+        "PyHDX Documentation",
         author,
         "pyhdx",
         "Derive ΔG for single residues from HDX-MS data",