conflict resolution

.github/workflows/run_tests.yaml

@@ -0,0 +1,47 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: chartboost/ruff-action@v1
+        name: ruff check
+        with:
+          args: 'check --output-format=github'
+      - uses: chartboost/ruff-action@v1
+        name: ruff format
+        with:
+          args: 'format --check'
+  tests:
+    name: unit tests (${{ matrix.python-version }}, ${{ matrix.os }})
+    runs-on:  ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      max-parallel: 6
+      matrix:
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
+        python-version: ["3.10", "3.12"]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install deps
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -e .
+    - name: Test with pytest
+      run: |
+        pytest -k 'test_'

.gitignore

@@ -162,4 +162,5 @@ cython_debug/
 #.idea/
 
 # local scripts
-src/local
+src/local
+src/analysis

requirements.txt

@@ -1,6 +1,6 @@
-ibllib
+ibllib @ git+https://github.com/int-brain-lab/ibllib@develop
 numpy
 matplotlib
 pytest
 scipy
-pandera
+pandera

src/gui/rawdata_visualizer.py

@@ -1,19 +1,23 @@
 import sys
-import pandas as pd
-import matplotlib.pyplot as plt
-from PyQt5.QtWidgets import QApplication, QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QFileDialog, QTableWidget, \
-    QTableWidgetItem, QComboBox, QGridLayout, QLineEdit
-from PyQt5.QtCore import Qt
+from PyQt5.QtWidgets import (
+    QApplication,
+    QWidget,
+    QVBoxLayout,
+    QHBoxLayout,
+    QPushButton,
+    QFileDialog,
+    QComboBox,
+    QGridLayout,
+    QLineEdit,
+)
 from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
 from matplotlib.backends.backend_qt5 import NavigationToolbar2QT as NavigationToolbar
-from pydantic.v1 import NoneStr
 
 from iblphotometry.io import from_raw_neurophotometrics
 import iblphotometry.plots as plots
 
 import iblphotometry.preprocessing as ffpr
 import numpy as np
-from matplotlib.colorbar import Colorbar
 
 
 class DataFrameVisualizerApp(QWidget):
@@ -75,7 +79,6 @@ def init_ui(self):
         main_layout.addLayout(time_layout)
         main_layout.addWidget(self.apply_button)
 
-
         # Set up plots layout
         self.plot_layout = QGridLayout()
         self.plotobj = plots.PlotSignal()
@@ -159,20 +162,21 @@ def apply_time_range(self):
                 start_time = float(start_time_str)
 
             if end_time_str == '':
-                end_time = self.times[len(self.times)-1]
+                end_time = self.times[len(self.times) - 1]
             else:
                 end_time = float(end_time_str)
             # Filter dataframe based on user input
             indx_time = (self.times >= start_time) & (self.times <= end_time)
 
-
             if len(indx_time) == 0:
-                print("No data in the specified range.")
+                print('No data in the specified range.')
             else:
                 self.plot_time_index = indx_time
                 self.update_plots()
         except ValueError:
-            print("Invalid time format. Please enter a valid time point in the format of a float.")
+            print(
+                'Invalid time format. Please enter a valid time point in the format of a float.'
+            )
 
     def update_column_selector(self):
         if self.df is not None:
@@ -185,11 +189,12 @@ def update_plots(self):
         selected_column = self.column_selector.currentText()
 
         if selected_column and self.df is not None:
-
             raw_signal = self.df[selected_column].values[self.plot_time_index]
             times = self.times[self.plot_time_index]
             if self.dfiso is not None:
-                raw_isosbestic = self.dfiso[selected_column].values[self.plot_time_index]
+                raw_isosbestic = self.dfiso[selected_column].values[
+                    self.plot_time_index
+                ]
             else:
                 raw_isosbestic = None
 
@@ -199,12 +204,16 @@ def update_plots(self):
             if self.filtered_df is None:
                 processed_signal = None
             else:
-                processed_signal = self.filtered_df[selected_column].values[self.plot_time_index]
-
-            self.plotobj.set_data(raw_signal=raw_signal,
-                                  times=times,
-                                  raw_isosbestic=raw_isosbestic,
-                                  processed_signal=processed_signal)
+                processed_signal = self.filtered_df[selected_column].values[
+                    self.plot_time_index
+                ]
+
+            self.plotobj.set_data(
+                raw_signal=raw_signal,
+                times=times,
+                raw_isosbestic=raw_isosbestic,
+                processed_signal=processed_signal,
+            )
             self.plotobj.raw_processed_figure2(self.axes)
 
             # Redraw the canvas
@@ -230,12 +239,11 @@ def apply_filter(self):
         # Get the selected filter option from the filter dropdown
         filter_option = self.filter_selector.currentText()
 
-        if filter_option == "Select Filter":
+        if filter_option == 'Select Filter':
             self.filtered_df = None
             # After applying the filter, update the plots
             self.update_plots()
 
-
         # Apply the appropriate filter to the dataframe and get the modified data
         if filter_option == 'Filter MAD':
             self.filtered_df = self.filter_mad(self.df)

src/iblphotometry/loaders.py

@@ -12,15 +12,15 @@ def __init__(self, one, verbose=False):
         self.one = one
         self.verbose = verbose
 
-    def load_photometry_data(self, eid=None, pid=None, rename=True) -> nap.TsdFrame:
+    def load_photometry_data(self, eid=None, pid=None, rename=True) -> pd.DataFrame:
         if pid is not None:
             raise NotImplementedError
             # return self._load_data_from_pid(pid)
 
         if eid is not None:
             return self._load_data_from_eid(eid, rename=rename)
 
-    def _load_data_from_eid(self, eid, rename=True) -> nap.TsdFrame:
+    def _load_data_from_eid(self, eid, rename=True) -> pd.DataFrame:
         raw_photometry_df = self.one.load_dataset(eid, 'photometry.signal.pqt')
         locations_df = self.one.load_dataset(eid, 'photometryROI.locations.pqt')
         read_config = dict(

src/iblphotometry/pipelines.py

@@ -3,7 +3,6 @@
 from iblphotometry.processing import remove_spikes, lowpass_bleachcorrect, isosbestic_correct, sliding_mad, zscore
 
 import logging
-from copy import copy
 
 logger = logging.getLogger()
 

src/iblphotometry/plots.py

@@ -17,7 +17,7 @@
 
 PSTH_EVENTS = {
     'feedback_times': 'Feedback',
-    'stimOnTrigger_times': 'Stim on',
+    'stimOn_times': 'Stim on',
     'firstMovement_times': 'First move',
 }
 
@@ -143,7 +143,9 @@ def raw_processed_figure2(self, axd):
 
 
 class PlotSignalResponse:
-    def __init__(self, trials, processed_signal, times, fs=None, event_window=np.array([-1, 2])):
+    def __init__(
+        self, trials, processed_signal, times, fs=None, event_window=np.array([-1, 2])
+    ):
         self.trials = trials
         self.times = times
         self.processed_signal = processed_signal
@@ -173,7 +175,7 @@ def compute_events_psth(self):
         psth_dict['times'] = psth_times(self.fs, self.event_window)
         return psth_dict
 
-    def update_psth_dict(self, event):     
+    def update_psth_dict(self, event):
         try:
             self.psth_dict[event], _ = psth(
                 self.processed_signal,
@@ -191,11 +193,7 @@ def plot_trialsort_psth(self):
         signal_keys = [k for k in self.psth_dict.keys() if k != 'times']
         for iaxs, event in enumerate(signal_keys):
             axs_plt = [axs[0, iaxs], axs[1, iaxs]]
-            plot_psth(
-                self.psth_dict[event],
-                self.psth_dict['times'],
-                axs=axs_plt
-            )
+            plot_psth(self.psth_dict[event], self.psth_dict['times'], axs=axs_plt)
             if event in PSTH_EVENTS.keys():
                 axs_plt[0].set_title(PSTH_EVENTS[event])
             else:
@@ -211,7 +209,7 @@ def plot_trialsort_psth(self):
         fig.tight_layout()
         return fig, axs
 
-    def plot_processed_trialtick(self, event_key='stimOnTrigger_times'):
+    def plot_processed_trialtick(self, event_key='stimOn_times'):
         fig, ax = plt.subplots(1, 1)
         plt.figure(figsize=(10, 6))
         events = self.trials[event_key]

src/tests/base_tests.py → src/iblphotometry_tests/base_tests.py

@@ -17,6 +17,8 @@ def get_fixtures() -> dict:
         / session_folder
         / Path('raw_photometry_data/_neurophotometrics_fpData.raw.pqt'),
         'raw_neurophotometrics_csv': data_folder / 'raw_photometry.csv',
+        'raw_kcenia_pqt': data_folder / 'raw_photometry.pqt',
+        'trials_table_kcenia_pqt': data_folder / '_ibl_trials.table.pqt',
         'trials_table_pqt': data_folder / session_folder / 'alf/_ibl_trials.table.pqt',
     }
 

src/tests/test_loaders.py → src/iblphotometry_tests/test_loaders.py

@@ -1,10 +1,9 @@
-import unittest
 import iblphotometry.io as fio
 import numpy as np
-import tests.base_tests
+from iblphotometry_tests.base_tests import PhotometryDataTestCase
 
 
-class TestLoaders(tests.base_tests.PhotometryDataTestCase):
+class TestLoaders(PhotometryDataTestCase):
     # think here about the possible use cases
 
     # to read from a .csv from disk

src/tests/test_metrics.py → src/iblphotometry_tests/test_metrics.py

@@ -1,12 +1,11 @@
-import unittest
 import iblphotometry.io as fio
 import iblphotometry.metrics as metrics
 import pandas as pd
 
-import tests.base_tests
+from iblphotometry_tests.base_tests import PhotometryDataTestCase
 
 
-class TestMetrics(tests.base_tests.PhotometryDataTestCase):
+class TestMetrics(PhotometryDataTestCase):
     # think here about the possible use cases
 
     def test_metrics(self):

src/iblphotometry_tests/test_pipelines.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+import iblphotometry.io as fio
+from iblphotometry.pipelines import (
+    run_pipeline,
+    sliding_mad_pipeline,
+    isosbestic_correction_pipeline,
+)
+from iblphotometry.synthetic import generate_dataframe
+from iblphotometry_tests.base_tests import PhotometryDataTestCase
+
+
+class TestPipelines(PhotometryDataTestCase):
+    def test_single_band_pipeline(self):
+        # on synthetic data
+        raw_dfs = generate_dataframe()
+        run_pipeline(sliding_mad_pipeline, raw_dfs['raw_calcium'])
+
+        Path(__file__).parent.joinpath()
+        # on real data
+        raw_dfs = fio.from_pqt(
+            self.paths['photometry_signal_pqt'],
+            self.paths['photometryROI_locations_pqt'],
+        )
+        signal_bands = list(raw_dfs.keys())
+        run_pipeline(sliding_mad_pipeline, raw_dfs[signal_bands[0]])
+
+    def test_isosbestic_pipeline(self):
+        # on synthetic data
+        raw_dfs = generate_dataframe()
+
+        # run pipeline
+        run_pipeline(
+            isosbestic_correction_pipeline,
+            raw_dfs['raw_calcium'],
+            raw_dfs['raw_isosbestic'],
+        )