refactoring and adding io function for new notebook

src/iblphotometry/io.py

@@ -45,7 +45,9 @@ def from_dataframe(
 
     # infer if not explicitly provided: defaults to everything that starts with 'Region'
     if data_columns is None:
-        data_columns = [col for col in raw_df.columns if col.startswith('Region')]
+        # this hacky parser currently deals with the inconsistency between carolinas and alejandros extraction
+        # https://github.com/int-brain-lab/ibl-photometry/issues/35
+        data_columns = [col for col in raw_df.columns if col.startswith('Region') or col.startswith('G')]
 
     # infer name of time column if not provided
     if time_column is None:
@@ -73,6 +75,19 @@ def from_dataframe(
 
     return raw_dfs
 
+def from_dataframes(raw_df: pd.DataFrame, locations_df: pd.DataFrame):
+    data_columns = (list(locations_df.index),)
+    rename = locations_df['brain_region'].to_dict()
+
+    read_config = dict(
+        data_columns=data_columns,
+        time_column='times',
+        channel_column='name',
+        rename=rename,
+    )
+
+    return from_dataframe(raw_df, **read_config)
+
 
 def from_pqt(
     signal_pqt_path: str | Path,
@@ -110,7 +125,7 @@ def from_pqt(
     return from_dataframe(raw_df, **read_config)
 
 
-def _read_raw_neurophotometrics_df(raw_df: pd.DataFrame, rois=None) -> pd.DataFrame:
+def from_raw_neurophotometrics_df(raw_df: pd.DataFrame, rois=None, drop_first=True) -> pd.DataFrame:
     """reads in parses the output of the neurophotometrics FP3002
 
     Args:
@@ -159,10 +174,14 @@ def _read_raw_neurophotometrics_df(raw_df: pd.DataFrame, rois=None) -> pd.DataFr
             for cn in ['name', 'color', 'wavelength']:
                 out_df.loc[states == state, cn] = channel_meta_map.iloc[ic[0]][cn]
 
+    # drop first frame
+    if drop_first:
+        out_df = out_df.iloc[1:].reset_index()
+
     return out_df
 
 
-def from_raw_neurophotometrics(
+def from_raw_neurophotometrics_file(
     path: str | Path,
     drop_first=True,
     validate=True,
@@ -181,8 +200,7 @@ def from_raw_neurophotometrics(
         nap.TsdFrame: _description_ # FIXME
     """
     warnings.warn(
-        'loading a photometry from raw neurophotometrics output. The data will _not_ be synced and\
-            is being split into channels by LedState (converted to LED wavelength in nm)'
+        'loading photometry from raw neurophotometrics output. The data will _not_ be synced and is being split into channels by LedState (converted to LED wavelength in nm)'
     )
     if isinstance(path, str):
         path = Path(path)
@@ -199,11 +217,11 @@ def from_raw_neurophotometrics(
     if validate:
         raw_df = _validate_dataframe(raw_df)
 
-    df = _read_raw_neurophotometrics_df(raw_df)
+    df = from_raw_neurophotometrics_df(raw_df)
 
     # drop first frame
     if drop_first:
-        df = df.iloc[1:]
+        df = df.iloc[1:].reset_index()
 
     data_columns = [col for col in df.columns if col.startswith('G')]
     read_config = dict(

src/iblphotometry/qc.py

@@ -1,25 +1,19 @@
 # %%
-import numpy as np
-from scipy.stats import linregress
-import pandas as pd
-
+import gc
+from collections.abc import Callable
 from tqdm import tqdm
 import logging
-
-import gc
-
-from iblphotometry.processing import make_sliding_window
-from pipelines import run_pipeline
 import warnings
 
-from brainbox.io.one import SessionLoader
+import numpy as np
+import pandas as pd
+from scipy.stats import linregress
 
-warnings.filterwarnings('once', category=DeprecationWarning, module='pynapple')
+from iblphotometry.processing import make_sliding_window
+from iblphotometry.pipelines import run_pipeline
 
 logger = logging.getLogger()
 
-from collections.abc import Callable
-
 # %% # those could be in metrics
 def sliding_metric(
     F: pd.Series,
@@ -119,9 +113,9 @@ def run_qc(
     for eid in tqdm(eids):
         print(eid)
         # get photometry data
-        raw_tfs = data_loader.load_photometry_data(eid=eid)
-        signal_bands = list(raw_tfs.keys())
-        brain_regions = raw_tfs[signal_bands[0]]
+        raw_dfs = data_loader.load_photometry_data(eid=eid)
+        signal_bands = list(raw_dfs.keys())
+        brain_regions = raw_dfs[signal_bands[0]]
 
         # get behavioral data
         # TODO this should be provided
@@ -136,7 +130,7 @@ def run_qc(
         # trials = data_loader.one.load_dataset(eid, '*trials.table.pqt')
 
         for band in signal_bands:
-            raw_tf = raw_tfs[band]
+            raw_tf = raw_dfs[band]
             for region in brain_regions:
                 qc_result = qc_series(
                     raw_tf[region], qc_metrics['raw'], sliding_kwargs=None, eid=eid
@@ -151,14 +145,14 @@ def run_qc(
             if 'reference' in sigref_mapping:  # this is for isosbestic pipelines
                 proc_tf = run_pipeline(
                     pipeline,
-                    raw_tfs[sigref_mapping['signal']],
-                    raw_tfs[sigref_mapping['reference']],
+                    raw_dfs[sigref_mapping['signal']],
+                    raw_dfs[sigref_mapping['reference']],
                 )
             else:
                 # FIXME this fails for true-multiband
                 # this hack works for single-band
                 # possible fix could be that signal could be a list
-                proc_tf = run_pipeline(pipeline, raw_tfs[sigref_mapping['signal']])
+                proc_tf = run_pipeline(pipeline, raw_dfs[sigref_mapping['signal']])
 
             for region in brain_regions:
                 # sliding qc of the processed data