From 5a196411a985f6ae94a27d287ad8b18290237bdf Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Sun, 29 Sep 2024 14:24:30 -0400 Subject: [PATCH 1/3] add custom spikeglx subconverter --- README.md | 2 + .../convert_brainwide_map_raw_only.py | 40 +++-------------- src/ibl_to_nwb/converters/__init__.py | 2 + .../converters/_ibl_spikeglx_converter.py | 44 +++++++++++++++++++ 4 files changed, 54 insertions(+), 34 deletions(-) create mode 100644 src/ibl_to_nwb/converters/_ibl_spikeglx_converter.py diff --git a/README.md b/README.md index 693a2d6..02584f7 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ An `Interface` reads a single data stream (such as DLC pose estimation) and crea The `Converter` orchestrates the conversion by combining multiple interfaces, and can also be used to add additional metadata to the NWB file. It is responsible for creating the NWB file saved to disk. +Occasionally, a sub-`Converter`, such as the `IBLSpikeGLXConverter`, will be used to handle the conversion of multiple data streams that is more complex than a single interface can handle; though these behave like other `Interfaces` with respect to the main orchestrating `Converter`. + ## Metadata Anywhere you see handwritten text in the NWB files that is meant to be human-readable, it is likely that it was copied from the public Google IBL documents and written in the `.yaml` files found in `src/ibl_to_nwb/_metadata`. diff --git a/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py b/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py index f695968..8aca44a 100644 --- a/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py +++ b/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py @@ -1,10 +1,8 @@ from pathlib import Path -from brainbox.io.one import EphysSessionLoader, SpikeSortingLoader -from neuroconv.datainterfaces import SpikeGLXRecordingInterface from one.api import ONE -from ibl_to_nwb.converters import BrainwideMapConverter +from ibl_to_nwb.converters import BrainwideMapConverter, IblSpikeglxConverter from ibl_to_nwb.datainterfaces import RawVideoInterface session_id = "d32876dd-8303-4720-8e7e-20678dc2fd71" @@ -28,39 +26,13 @@ ) # Specify the path to the SpikeGLX files on the server but use ONE API for timestamps -probe_1_source_folder_path = Path("D:/example_data/ephy_testing_data/spikeglx/Noise4Sam_g0") -probe_2_source_folder_path = Path( - "D:/example_data/ephy_testing_data/spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g0/" -) - - -# Initialize interfaces one probe at a time and properly align raw timestamps -ephys_session_loader = EphysSessionLoader(one=ibl_client, eid=session_id) - -probe_to_imec_map = { - "probe00": 0, - "probe01": 1, -} - -data_interfaces = list() -for probe_name, pid in ephys_session_loader.probes.items(): - spike_sorting_loader = SpikeSortingLoader(pid=pid, one=ibl_client) - - probe_index = probe_to_imec_map[probe_name] - for band in ["ap", "lf"]: - file_path = probe_1_source_folder_path / f"Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec{probe_index}.{band}.bin" - interface = SpikeGLXRecordingInterface(file_path=file_path) - - # This is the syntax for aligning the raw timestamps; I cannot test this without the actual data as stored - # on your end, so please work with Heberto if there are any problems after uncommenting - - # band_info = spike_sorting_loader.raw_electrophysiology(band=band, stream=True) - # aligned_timestamps = spike_sorting_loader.samples2times(numpy.arange(0, band_info.ns), direction='forward') - # interface.set_aligned_timestamps(aligned_timestamps=aligned_timestamps) +data_interfaces = [] - data_interfaces.append(interface) +spikeglx_source_folder_path = Path("D:/example_data/ephy_testing_data/spikeglx/Noise4Sam_g0") +spikeglx_subconverter = IblSpikeglxConverter(folder_path=spikeglx_source_folder_path) +data_interfaces.append(spikeglx_subconverter) -# Raw video take some special handling +# Raw video takes some special handling metadata_retrieval = BrainwideMapConverter(one=ibl_client, session=session_id, data_interfaces=[], verbose=False) subject_id = metadata_retrieval.get_metadata()["Subject"]["subject_id"] diff --git a/src/ibl_to_nwb/converters/__init__.py b/src/ibl_to_nwb/converters/__init__.py index 936d87c..d95d370 100644 --- a/src/ibl_to_nwb/converters/__init__.py +++ b/src/ibl_to_nwb/converters/__init__.py @@ -1,7 +1,9 @@ from ._brainwide_map_converter import BrainwideMapConverter from ._iblconverter import IblConverter +from ._ibl_spikeglx_converter import IblSpikeglxConverter __all__ = [ "BrainwideMapConverter", "IblConverter", + "IblSpikeglxConverter", ] diff --git a/src/ibl_to_nwb/converters/_ibl_spikeglx_converter.py b/src/ibl_to_nwb/converters/_ibl_spikeglx_converter.py new file mode 100644 index 0000000..f670466 --- /dev/null +++ b/src/ibl_to_nwb/converters/_ibl_spikeglx_converter.py @@ -0,0 +1,44 @@ +import one +from neuroconv.converters import SpikeGLXConverterPipe +from pydantic import DirectoryPath +from pynwb import NWBFile + + +class IBLSpikeGLXConverter(SpikeGLXConverterPipe): + + def __init__(self, folder_path: DirectoryPath, one: one.ONE) -> None: + super().__init__(folder_path=folder_path) + self.one = one + + def temporally_align_data_interfaces(self) -> None: + """Align the raw data timestamps to the other data streams using the ONE API.""" + # This is the syntax for aligning the raw timestamps; I cannot test this without the actual data as stored + # on your end, so please work with Heberto if there are any problems after uncommenting + # probe_to_imec_map = { + # "probe00": 0, + # "probe01": 1, + # } + # + # ephys_session_loader = EphysSessionLoader(one=self.one, eid=session_id) + # probes = ephys_session_loader.probes + # for probe_name, pid in ephys_session_loader.probes.items(): + # spike_sorting_loader = SpikeSortingLoader(pid=pid, one=ibl_client) + # + # probe_index = probe_to_imec_map[probe_name] + # for band in ["ap", "lf"]: + # recording_interface = next( + # interface + # for interface in self.data_interface_objects + # if f"imec{probe_index}.{band}" in interface.source_data["file_path"] + # ) + # + # band_info = spike_sorting_loader.raw_electrophysiology(band=band, stream=True) + # aligned_timestamps = spike_sorting_loader.samples2times(numpy.arange(0, band_info.ns), direction='forward') + # recording_interface.set_aligned_timestamps(aligned_timestamps=aligned_timestamps) + pass + + def add_to_nwbfile(self, nwbfile: NWBFile, metadata) -> None: + self.temporally_align_data_interfaces() + super().add_to_nwbfile(nwbfile=nwbfile, metadata=metadata) + + # TODO: Add ndx-extracellular-ephys here From 00ac11e6905d2338bcac4bd423ba0daaaff45048 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Sun, 29 Sep 2024 14:26:11 -0400 Subject: [PATCH 2/3] add custom spikeglx subconverter --- src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py | 2 +- src/ibl_to_nwb/converters/__init__.py | 4 ++-- src/ibl_to_nwb/converters/_ibl_spikeglx_converter.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py b/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py index 8aca44a..0b6f24a 100644 --- a/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py +++ b/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py @@ -29,7 +29,7 @@ data_interfaces = [] spikeglx_source_folder_path = Path("D:/example_data/ephy_testing_data/spikeglx/Noise4Sam_g0") -spikeglx_subconverter = IblSpikeglxConverter(folder_path=spikeglx_source_folder_path) +spikeglx_subconverter = IblSpikeglxConverter(folder_path=spikeglx_source_folder_path, one=ibl_client) data_interfaces.append(spikeglx_subconverter) # Raw video takes some special handling diff --git a/src/ibl_to_nwb/converters/__init__.py b/src/ibl_to_nwb/converters/__init__.py index d95d370..71fe855 100644 --- a/src/ibl_to_nwb/converters/__init__.py +++ b/src/ibl_to_nwb/converters/__init__.py @@ -1,9 +1,9 @@ from ._brainwide_map_converter import BrainwideMapConverter from ._iblconverter import IblConverter -from ._ibl_spikeglx_converter import IblSpikeglxConverter +from ._ibl_spikeglx_converter import IblSpikeGlxConverter __all__ = [ "BrainwideMapConverter", "IblConverter", - "IblSpikeglxConverter", + "IblSpikeGlxConverter", ] diff --git a/src/ibl_to_nwb/converters/_ibl_spikeglx_converter.py b/src/ibl_to_nwb/converters/_ibl_spikeglx_converter.py index f670466..3f9cc9a 100644 --- a/src/ibl_to_nwb/converters/_ibl_spikeglx_converter.py +++ b/src/ibl_to_nwb/converters/_ibl_spikeglx_converter.py @@ -1,12 +1,12 @@ -import one from neuroconv.converters import SpikeGLXConverterPipe +from one.api import ONE from pydantic import DirectoryPath from pynwb import NWBFile -class IBLSpikeGLXConverter(SpikeGLXConverterPipe): +class IblSpikeGlxConverter(SpikeGLXConverterPipe): - def __init__(self, folder_path: DirectoryPath, one: one.ONE) -> None: + def __init__(self, folder_path: DirectoryPath, one: ONE) -> None: super().__init__(folder_path=folder_path) self.one = one From 9d72fd0d97df0cd5bcbb7a0b443725eecd2d81ad Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Sun, 29 Sep 2024 14:27:08 -0400 Subject: [PATCH 3/3] add custom spikeglx subconverter --- src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py b/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py index 0b6f24a..09388d1 100644 --- a/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py +++ b/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py @@ -2,7 +2,7 @@ from one.api import ONE -from ibl_to_nwb.converters import BrainwideMapConverter, IblSpikeglxConverter +from ibl_to_nwb.converters import BrainwideMapConverter, IblSpikeGlxConverter from ibl_to_nwb.datainterfaces import RawVideoInterface session_id = "d32876dd-8303-4720-8e7e-20678dc2fd71" @@ -29,7 +29,7 @@ data_interfaces = [] spikeglx_source_folder_path = Path("D:/example_data/ephy_testing_data/spikeglx/Noise4Sam_g0") -spikeglx_subconverter = IblSpikeglxConverter(folder_path=spikeglx_source_folder_path, one=ibl_client) +spikeglx_subconverter = IblSpikeGlxConverter(folder_path=spikeglx_source_folder_path, one=ibl_client) data_interfaces.append(spikeglx_subconverter) # Raw video takes some special handling