From 535626302a6c85945beb07288c48b76887947517 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 11 Dec 2024 12:44:21 -0600 Subject: [PATCH] Support digital channels on NIDQ interface and use TimeSeries instead of ElecricalSeries for analog channels (#1152) Co-authored-by: Ben Dichter --- CHANGELOG.md | 7 +- .../recording/spikeglx.rst | 2 +- .../ecephys/spikeglx/spikeglxdatainterface.py | 8 +- .../ecephys/spikeglx/spikeglxnidqinterface.py | 271 +++++++++++++++--- .../tools/testing/mock_interfaces.py | 4 + .../test_ecephys/test_mock_nidq_interface.py | 52 +--- .../spikeglx_single_probe_metadata.json | 16 +- .../ecephys/test_aux_interfaces.py | 100 ------- .../ecephys/test_nidq_interface.py | 57 ++++ .../ecephys/test_spikeglx_converter.py | 27 +- .../test_temporal_alignment_methods.py | 1 - 11 files changed, 327 insertions(+), 218 deletions(-) delete mode 100644 tests/test_on_data/ecephys/test_aux_interfaces.py create mode 100644 tests/test_on_data/ecephys/test_nidq_interface.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 728123253..c9c9afcde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ * Make `NWBMetaDataEncoder` public again [PR #1142](https://github.com/catalystneuro/neuroconv/pull/1142) * Fix a bug where data in `DeepLabCutInterface` failed to write when `ndx-pose` was not imported. [#1144](https://github.com/catalystneuro/neuroconv/pull/1144) * `SpikeGLXConverterPipe` converter now accepts multi-probe structures with multi-trigger and does not assume a specific folder structure [#1150](https://github.com/catalystneuro/neuroconv/pull/1150) +* `SpikeGLXNIDQInterface` is no longer written as an ElectricalSeries [#1152](https://github.com/catalystneuro/neuroconv/pull/1152) + ## Features * Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124) @@ -17,7 +19,10 @@ * `SpikeGLXRecordingInterface` now also accepts `folder_path` making its behavior equivalent to SpikeInterface [#1150](https://github.com/catalystneuro/neuroconv/pull/1150) * Added the `rclone_transfer_batch_job` helper function for executing Rclone data transfers in AWS Batch jobs. [PR #1085](https://github.com/catalystneuro/neuroconv/pull/1085) * Added the `deploy_neuroconv_batch_job` helper function for deploying NeuroConv AWS Batch jobs. [PR #1086](https://github.com/catalystneuro/neuroconv/pull/1086) -* YAML specification files now accept an outer keyword `upload_to_dandiset="< six-digit ID >"` to automatically upload the produced NWB files to the DANDI archive [PR #1089](https://github.com/catalystneuro/neuroconv/pull/1089) +* YAML specification files now accepts an outer keyword `upload_to_dandiset="< six-digit ID >"` to automatically upload the produced NWB files to the DANDI archive [PR #1089](https://github.com/catalystneuro/neuroconv/pull/1089) +*`SpikeGLXNIDQInterface` now handdles digital demuxed channels (`XD0`) [#1152](https://github.com/catalystneuro/neuroconv/pull/1152) + + ## Improvements diff --git a/docs/conversion_examples_gallery/recording/spikeglx.rst b/docs/conversion_examples_gallery/recording/spikeglx.rst index 7f57470af..97b23bac9 100644 --- a/docs/conversion_examples_gallery/recording/spikeglx.rst +++ b/docs/conversion_examples_gallery/recording/spikeglx.rst @@ -24,7 +24,7 @@ We can easily convert all data stored in the native SpikeGLX folder structure to >>> >>> folder_path = f"{ECEPHY_DATA_PATH}/spikeglx/Noise4Sam_g0" >>> converter = SpikeGLXConverterPipe(folder_path=folder_path) - >>> + Source data is valid! >>> # Extract what metadata we can from the source files >>> metadata = converter.get_metadata() >>> # For data provenance we add the time zone information to the conversion diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index 00419e036..e8b6a78c9 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -66,7 +66,7 @@ def __init__( Folder path containing the binary files of the SpikeGLX recording. stream_id: str, optional Stream ID of the SpikeGLX recording. - Examples are 'nidq', 'imec0.ap', 'imec0.lf', 'imec1.ap', 'imec1.lf', etc. + Examples are 'imec0.ap', 'imec0.lf', 'imec1.ap', 'imec1.lf', etc. file_path : FilePathType Path to .bin file. Point to .ap.bin for SpikeGLXRecordingInterface and .lf.bin for SpikeGLXLFPInterface. verbose : bool, default: True @@ -74,10 +74,14 @@ def __init__( es_key : str, the key to access the metadata of the ElectricalSeries. """ + if stream_id == "nidq": + raise ValueError( + "SpikeGLXRecordingInterface is not designed to handle nidq files. Use SpikeGLXNIDQInterface instead" + ) + if file_path is not None and stream_id is None: self.stream_id = fetch_stream_id_for_spikelgx_file(file_path) self.folder_path = Path(file_path).parent - else: self.stream_id = stream_id self.folder_path = Path(folder_path) diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py index 1d7079716..5249dfe39 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py @@ -1,39 +1,36 @@ +import warnings from pathlib import Path -from typing import Optional +from typing import Literal, Optional import numpy as np from pydantic import ConfigDict, DirectoryPath, FilePath, validate_call +from pynwb import NWBFile +from pynwb.base import TimeSeries from .spikeglx_utils import get_session_start_time -from ..baserecordingextractorinterface import BaseRecordingExtractorInterface +from ....basedatainterface import BaseDataInterface from ....tools.signal_processing import get_rising_frames_from_ttl -from ....utils import get_json_schema_from_method_signature +from ....tools.spikeinterface.spikeinterface import _recording_traces_to_hdmf_iterator +from ....utils import ( + calculate_regular_series_rate, + get_json_schema_from_method_signature, +) -class SpikeGLXNIDQInterface(BaseRecordingExtractorInterface): +class SpikeGLXNIDQInterface(BaseDataInterface): """Primary data interface class for converting the high-pass (ap) SpikeGLX format.""" display_name = "NIDQ Recording" - keywords = BaseRecordingExtractorInterface.keywords + ("Neuropixels",) + keywords = ("Neuropixels", "nidq", "NIDQ", "SpikeGLX") associated_suffixes = (".nidq", ".meta", ".bin") info = "Interface for NIDQ board recording data." - ExtractorName = "SpikeGLXRecordingExtractor" - stream_id = "nidq" - @classmethod def get_source_schema(cls) -> dict: source_schema = get_json_schema_from_method_signature(method=cls.__init__, exclude=["x_pitch", "y_pitch"]) source_schema["properties"]["file_path"]["description"] = "Path to SpikeGLX .nidq file." return source_schema - def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: - - extractor_kwargs = source_data.copy() - extractor_kwargs["folder_path"] = self.folder_path - extractor_kwargs["stream_id"] = self.stream_id - return extractor_kwargs - @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def __init__( self, @@ -56,12 +53,18 @@ def __init__( Path to .nidq.bin file. verbose : bool, default: True Whether to output verbose text. - load_sync_channel : bool, default: False - Whether to load the last channel in the stream, which is typically used for synchronization. - If True, then the probe is not loaded. es_key : str, default: "ElectricalSeriesNIDQ" """ + if load_sync_channel: + + warnings.warn( + "The 'load_sync_channel' parameter is deprecated and will be removed in June 2025. " + "The sync channel data is only available the raw files of spikeglx`.", + DeprecationWarning, + stacklevel=2, + ) + if file_path is None and folder_path is None: raise ValueError("Either 'file_path' or 'folder_path' must be provided.") @@ -72,18 +75,36 @@ def __init__( if folder_path is not None: self.folder_path = Path(folder_path) + from spikeinterface.extractors import SpikeGLXRecordingExtractor + + self.recording_extractor = SpikeGLXRecordingExtractor( + folder_path=self.folder_path, + stream_id="nidq", + all_annotations=True, + ) + + channel_ids = self.recording_extractor.get_channel_ids() + analog_channel_signatures = ["XA", "MA"] + self.analog_channel_ids = [ch for ch in channel_ids if "XA" in ch or "MA" in ch] + self.has_analog_channels = len(self.analog_channel_ids) > 0 + self.has_digital_channels = len(self.analog_channel_ids) < len(channel_ids) + if self.has_digital_channels: + import ndx_events # noqa: F401 + from spikeinterface.extractors import SpikeGLXEventExtractor + + self.event_extractor = SpikeGLXEventExtractor(folder_path=self.folder_path) + super().__init__( verbose=verbose, load_sync_channel=load_sync_channel, es_key=es_key, + folder_path=self.folder_path, + file_path=file_path, ) - self.source_data.update(file_path=str(file_path)) - self.recording_extractor.set_property( - key="group_name", values=["NIDQChannelGroup"] * self.recording_extractor.get_num_channels() - ) + self.subset_channels = None - signal_info_key = (0, self.stream_id) # Key format is (segment_index, stream_id) + signal_info_key = (0, "nidq") # Key format is (segment_index, stream_id) self._signals_info_dict = self.recording_extractor.neo_reader.signals_info_dict[signal_info_key] self.meta = self._signals_info_dict["meta"] @@ -101,24 +122,206 @@ def get_metadata(self) -> dict: manufacturer="National Instruments", ) - # Add groups metadata - metadata["Ecephys"]["Device"] = [device] + metadata["Devices"] = [device] - metadata["Ecephys"]["ElectrodeGroup"][0].update( - name="NIDQChannelGroup", description="A group representing the NIDQ channels.", device=device["name"] - ) - metadata["Ecephys"]["Electrodes"] = [ - dict(name="group_name", description="Name of the ElectrodeGroup this electrode is a part of."), - ] - metadata["Ecephys"]["ElectricalSeriesNIDQ"][ - "description" - ] = "Raw acquisition traces from the NIDQ (.nidq.bin) channels." return metadata def get_channel_names(self) -> list[str]: """Return a list of channel names as set in the recording extractor.""" return list(self.recording_extractor.get_channel_ids()) + def add_to_nwbfile( + self, + nwbfile: NWBFile, + metadata: Optional[dict] = None, + stub_test: bool = False, + starting_time: Optional[float] = None, + write_as: Literal["raw", "lfp", "processed"] = "raw", + write_electrical_series: bool = True, + iterator_type: Optional[str] = "v2", + iterator_opts: Optional[dict] = None, + always_write_timestamps: bool = False, + ): + """ + Add NIDQ board data to an NWB file, including both analog and digital channels if present. + + Parameters + ---------- + nwbfile : NWBFile + The NWB file to which the NIDQ data will be added + metadata : Optional[dict], default: None + Metadata dictionary with device information. If None, uses default metadata + stub_test : bool, default: False + If True, only writes a small amount of data for testing + starting_time : Optional[float], default: None + DEPRECATED: Will be removed in June 2025. Starting time offset for the TimeSeries + write_as : Literal["raw", "lfp", "processed"], default: "raw" + DEPRECATED: Will be removed in June 2025. Specifies how to write the data + write_electrical_series : bool, default: True + DEPRECATED: Will be removed in June 2025. Whether to write electrical series data + iterator_type : Optional[str], default: "v2" + Type of iterator to use for data streaming + iterator_opts : Optional[dict], default: None + Additional options for the iterator + always_write_timestamps : bool, default: False + If True, always writes timestamps instead of using sampling rate + """ + + if starting_time is not None: + warnings.warn( + "The 'starting_time' parameter is deprecated and will be removed in June 2025. " + "Use the time alignment methods for modifying the starting time or timestamps " + "of the data if needed: " + "https://neuroconv.readthedocs.io/en/main/user_guide/temporal_alignment.html", + DeprecationWarning, + stacklevel=2, + ) + + if write_as != "raw": + warnings.warn( + "The 'write_as' parameter is deprecated and will be removed in June 2025. " + "NIDQ should always be written in the acquisition module of NWB. " + "Writing data as LFP or processed data is not supported.", + DeprecationWarning, + stacklevel=2, + ) + + if write_electrical_series is not True: + warnings.warn( + "The 'write_electrical_series' parameter is deprecated and will be removed in June 2025. " + "The option to skip the addition of the data is no longer supported. " + "This option was used in ElectricalSeries to write the electrode and electrode group " + "metadata without the raw data.", + DeprecationWarning, + stacklevel=2, + ) + + if stub_test or self.subset_channels is not None: + recording = self.subset_recording(stub_test=stub_test) + else: + recording = self.recording_extractor + + if metadata is None: + metadata = self.get_metadata() + + # Add devices + device_metadata = metadata.get("Devices", []) + for device in device_metadata: + if device["name"] not in nwbfile.devices: + nwbfile.create_device(**device) + + # Add analog and digital channels + if self.has_analog_channels: + self._add_analog_channels( + nwbfile=nwbfile, + recording=recording, + iterator_type=iterator_type, + iterator_opts=iterator_opts, + always_write_timestamps=always_write_timestamps, + ) + + if self.has_digital_channels: + self._add_digital_channels(nwbfile=nwbfile) + + def _add_analog_channels( + self, + nwbfile: NWBFile, + recording, + iterator_type: Optional[str], + iterator_opts: Optional[dict], + always_write_timestamps: bool, + ): + """ + Add analog channels from the NIDQ board to the NWB file. + + Parameters + ---------- + nwbfile : NWBFile + The NWB file to add the analog channels to + recording : BaseRecording + The recording extractor containing the analog channels + iterator_type : Optional[str] + Type of iterator to use for data streaming + iterator_opts : Optional[dict] + Additional options for the iterator + always_write_timestamps : bool + If True, always writes timestamps instead of using sampling rate + """ + analog_recorder = recording.select_channels(channel_ids=self.analog_channel_ids) + channel_names = analog_recorder.get_property(key="channel_names") + segment_index = 0 + analog_data_iterator = _recording_traces_to_hdmf_iterator( + recording=analog_recorder, + segment_index=segment_index, + iterator_type=iterator_type, + iterator_opts=iterator_opts, + ) + + name = "TimeSeriesNIDQ" + description = f"Analog data from the NIDQ board. Channels are {channel_names} in that order." + time_series_kwargs = dict(name=name, data=analog_data_iterator, unit="a.u.", description=description) + + if always_write_timestamps: + timestamps = recording.get_times(segment_index=segment_index) + shifted_timestamps = timestamps + time_series_kwargs.update(timestamps=shifted_timestamps) + else: + recording_has_timestamps = recording.has_time_vector(segment_index=segment_index) + if recording_has_timestamps: + timestamps = recording.get_times(segment_index=segment_index) + rate = calculate_regular_series_rate(series=timestamps) + recording_t_start = timestamps[0] + else: + rate = recording.get_sampling_frequency() + recording_t_start = recording._recording_segments[segment_index].t_start or 0 + + if rate: + starting_time = float(recording_t_start) + time_series_kwargs.update(starting_time=starting_time, rate=recording.get_sampling_frequency()) + else: + shifted_timestamps = timestamps + time_series_kwargs.update(timestamps=shifted_timestamps) + + time_series = TimeSeries(**time_series_kwargs) + nwbfile.add_acquisition(time_series) + + def _add_digital_channels(self, nwbfile: NWBFile): + """ + Add digital channels from the NIDQ board to the NWB file as events. + + Parameters + ---------- + nwbfile : NWBFile + The NWB file to add the digital channels to + """ + from ndx_events import LabeledEvents + + event_channels = self.event_extractor.channel_ids + for channel_id in event_channels: + events_structure = self.event_extractor.get_events(channel_id=channel_id) + timestamps = events_structure["time"] + labels = events_structure["label"] + + # Some channels have no events + if timestamps.size > 0: + + # Timestamps are not ordered, the ones for off are first and then the ones for on + ordered_indices = np.argsort(timestamps) + ordered_timestamps = timestamps[ordered_indices] + ordered_labels = labels[ordered_indices] + + unique_labels = np.unique(ordered_labels) + label_to_index = {label: index for index, label in enumerate(unique_labels)} + data = [label_to_index[label] for label in ordered_labels] + + channel_name = channel_id.split("#")[-1] + description = f"On and Off Events from channel {channel_name}" + name = f"EventsNIDQDigitalChannel{channel_name}" + labeled_events = LabeledEvents( + name=name, description=description, timestamps=ordered_timestamps, data=data, labels=unique_labels + ) + nwbfile.add_acquisition(labeled_events) + def get_event_times_from_ttl(self, channel_name: str) -> np.ndarray: """ Return the start of event times from the rising part of TTL pulses on one of the NIDQ channels. diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 0652284e7..5a96b4b68 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -119,6 +119,9 @@ def __init__( """ from spikeinterface.extractors import NumpyRecording + self.has_analog_channels = True + self.has_digital_channels = False + if ttl_times is None: # Begin in 'off' state number_of_periods = int(np.ceil((signal_duration - ttl_duration) / (ttl_duration * 2))) @@ -127,6 +130,7 @@ def __init__( number_of_channels = len(ttl_times) channel_ids = [f"nidq#XA{channel_index}" for channel_index in range(number_of_channels)] # NIDQ channel IDs channel_groups = ["NIDQChannelGroup"] * number_of_channels + self.analog_channel_ids = channel_ids sampling_frequency = 25_000.0 # NIDQ sampling rate number_of_frames = int(signal_duration * sampling_frequency) diff --git a/tests/test_ecephys/test_mock_nidq_interface.py b/tests/test_ecephys/test_mock_nidq_interface.py index c0fb4eed2..1b098e1bb 100644 --- a/tests/test_ecephys/test_mock_nidq_interface.py +++ b/tests/test_ecephys/test_mock_nidq_interface.py @@ -1,4 +1,3 @@ -import pathlib from datetime import datetime from numpy.testing import assert_array_almost_equal @@ -46,47 +45,26 @@ def test_mock_metadata(): metadata = interface.get_metadata() - expected_ecephys_metadata = { - "Ecephys": { - "Device": [ - { - "name": "NIDQBoard", - "description": "A NIDQ board used in conjunction with SpikeGLX.", - "manufacturer": "National Instruments", - }, - ], - "ElectrodeGroup": [ - { - "name": "NIDQChannelGroup", - "description": "A group representing the NIDQ channels.", - "device": "NIDQBoard", - "location": "unknown", - }, - ], - "Electrodes": [ - {"name": "group_name", "description": "Name of the ElectrodeGroup this electrode is a part of."} - ], - "ElectricalSeriesNIDQ": { - "name": "ElectricalSeriesNIDQ", - "description": "Raw acquisition traces from the NIDQ (.nidq.bin) channels.", - }, - } - } - - assert metadata["Ecephys"] == expected_ecephys_metadata["Ecephys"] + expected_devices_metadata = [ + { + "name": "NIDQBoard", + "description": "A NIDQ board used in conjunction with SpikeGLX.", + "manufacturer": "National Instruments", + }, + ] + + assert metadata["Devices"] == expected_devices_metadata expected_start_time = datetime(2020, 11, 3, 10, 35, 10) assert metadata["NWBFile"]["session_start_time"] == expected_start_time -def test_mock_run_conversion(tmpdir: pathlib.Path): +def test_mock_run_conversion(tmp_path): interface = MockSpikeGLXNIDQInterface() metadata = interface.get_metadata() - test_directory = pathlib.Path(tmpdir) / "TestMockSpikeGLXNIDQInterface" - test_directory.mkdir(exist_ok=True) - nwbfile_path = test_directory / "test_mock_run_conversion.nwb" + nwbfile_path = tmp_path / "test_mock_run_conversion.nwb" interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True) with NWBHDF5IO(path=nwbfile_path, mode="r") as io: @@ -94,11 +72,3 @@ def test_mock_run_conversion(tmpdir: pathlib.Path): assert "NIDQBoard" in nwbfile.devices assert len(nwbfile.devices) == 1 - - assert "NIDQChannelGroup" in nwbfile.electrode_groups - assert len(nwbfile.electrode_groups) == 1 - - assert list(nwbfile.electrodes.id[:]) == [0, 1, 2, 3, 4, 5, 6, 7] - - assert "ElectricalSeriesNIDQ" in nwbfile.acquisition - assert len(nwbfile.acquisition) == 1 diff --git a/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json b/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json index 20f11742b..f3f5fb595 100644 --- a/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json +++ b/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json @@ -8,11 +8,6 @@ "name": "NeuropixelImec0", "description": "{\"probe_type\": \"0\", \"probe_type_description\": \"NP1.0\", \"flex_part_number\": \"NP2_FLEX_0\", \"connected_base_station_part_number\": \"NP2_QBSC_00\"}", "manufacturer": "Imec" - }, - { - "name": "NIDQBoard", - "description": "A NIDQ board used in conjunction with SpikeGLX.", - "manufacturer": "National Instruments" } ], "ElectrodeGroup": [ @@ -21,13 +16,8 @@ "description": "A group representing probe/shank 'Imec0'.", "location": "unknown", "device": "NeuropixelImec0" - }, - { - "name": "NIDQChannelGroup", - "description": "A group representing the NIDQ channels.", - "location": "unknown", - "device": "NIDQBoard" } + ], "ElectricalSeriesAP": { "name": "ElectricalSeriesAP", @@ -47,10 +37,6 @@ "description": "The id of the contact on the electrode" } ], - "ElectricalSeriesNIDQ": { - "name": "ElectricalSeriesNIDQ", - "description": "Raw acquisition traces from the NIDQ (.nidq.bin) channels." - }, "ElectricalSeriesLF": { "name": "ElectricalSeriesLF", "description": "Acquisition traces for the ElectricalSeriesLF." diff --git a/tests/test_on_data/ecephys/test_aux_interfaces.py b/tests/test_on_data/ecephys/test_aux_interfaces.py deleted file mode 100644 index 7934e29a1..000000000 --- a/tests/test_on_data/ecephys/test_aux_interfaces.py +++ /dev/null @@ -1,100 +0,0 @@ -import unittest -from datetime import datetime - -import pytest -from parameterized import param, parameterized -from spikeinterface.core.testing import check_recordings_equal -from spikeinterface.extractors import NwbRecordingExtractor - -from neuroconv import NWBConverter -from neuroconv.datainterfaces import SpikeGLXNIDQInterface - -# enable to run locally in interactive mode -try: - from ..setup_paths import ECEPHY_DATA_PATH as DATA_PATH - from ..setup_paths import OUTPUT_PATH -except ImportError: - from setup_paths import ECEPHY_DATA_PATH as DATA_PATH - from setup_paths import OUTPUT_PATH - -if not DATA_PATH.exists(): - pytest.fail(f"No folder found in location: {DATA_PATH}!") - - -def custom_name_func(testcase_func, param_num, param): - interface_name = param.kwargs["data_interface"].__name__ - reduced_interface_name = interface_name.replace("Interface", "") - - return ( - f"{testcase_func.__name__}_{param_num}_" - f"{parameterized.to_safe_name(reduced_interface_name)}" - f"_{param.kwargs.get('case_name', '')}" - ) - - -class TestEcephysAuxNwbConversions(unittest.TestCase): - savedir = OUTPUT_PATH - - parameterized_aux_list = [ - param( - data_interface=SpikeGLXNIDQInterface, - interface_kwargs=dict(file_path=str(DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_t0.nidq.bin")), - case_name="load_sync_channel_False", - ), - param( - data_interface=SpikeGLXNIDQInterface, - interface_kwargs=dict( - file_path=str(DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_t0.nidq.bin"), - load_sync_channel=True, - ), - case_name="load_sync_channel_True", - ), - ] - - @parameterized.expand(input=parameterized_aux_list, name_func=custom_name_func) - def test_aux_recording_extractor_to_nwb(self, data_interface, interface_kwargs, case_name=""): - nwbfile_path = str(self.savedir / f"{data_interface.__name__}_{case_name}.nwb") - - class TestConverter(NWBConverter): - data_interface_classes = dict(TestAuxRecording=data_interface) - - converter = TestConverter(source_data=dict(TestAuxRecording=interface_kwargs)) - - for interface_kwarg in interface_kwargs: - if interface_kwarg in ["file_path", "folder_path"]: - self.assertIn( - member=interface_kwarg, container=converter.data_interface_objects["TestAuxRecording"].source_data - ) - - metadata = converter.get_metadata() - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - converter.run_conversion(nwbfile_path=nwbfile_path, overwrite=True, metadata=metadata) - recording = converter.data_interface_objects["TestAuxRecording"].recording_extractor - - electrical_series_name = metadata["Ecephys"][converter.data_interface_objects["TestAuxRecording"].es_key][ - "name" - ] - - # NWBRecordingExtractor on spikeinterface does not yet support loading data written from multiple segments. - if recording.get_num_segments() == 1: - # Spikeinterface behavior is to load the electrode table channel_name property as a channel_id - nwb_recording = NwbRecordingExtractor(file_path=nwbfile_path, electrical_series_name=electrical_series_name) - if "channel_name" in recording.get_property_keys(): - renamed_channel_ids = recording.get_property("channel_name") - else: - renamed_channel_ids = recording.get_channel_ids().astype("str") - recording = recording.channel_slice( - channel_ids=recording.get_channel_ids(), renamed_channel_ids=renamed_channel_ids - ) - - # Edge case that only occurs in testing; I think it's fixed in > 0.96.1 versions (unreleased as of 1/11/23) - # The NwbRecordingExtractor on spikeinterface experiences an issue when duplicated channel_ids - # are specified, which occurs during check_recordings_equal when there is only one channel - if nwb_recording.get_channel_ids()[0] != nwb_recording.get_channel_ids()[-1]: - check_recordings_equal(RX1=recording, RX2=nwb_recording, return_scaled=False) - if recording.has_scaled_traces() and nwb_recording.has_scaled_traces(): - check_recordings_equal(RX1=recording, RX2=nwb_recording, return_scaled=True) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_on_data/ecephys/test_nidq_interface.py b/tests/test_on_data/ecephys/test_nidq_interface.py new file mode 100644 index 000000000..6d6517323 --- /dev/null +++ b/tests/test_on_data/ecephys/test_nidq_interface.py @@ -0,0 +1,57 @@ +import numpy as np +import pytest +from pynwb import NWBHDF5IO + +from neuroconv.datainterfaces import SpikeGLXNIDQInterface + +# enable to run locally in interactive mode +try: + from ..setup_paths import ECEPHY_DATA_PATH +except ImportError: + from setup_paths import ECEPHY_DATA_PATH + +if not ECEPHY_DATA_PATH.exists(): + pytest.fail(f"No folder found in location: {ECEPHY_DATA_PATH}!") + + +def test_nidq_interface_digital_data(tmp_path): + + nwbfile_path = tmp_path / "nidq_test_digital.nwb" + folder_path = ECEPHY_DATA_PATH / "spikeglx" / "DigitalChannelTest_g0" + interface = SpikeGLXNIDQInterface(folder_path=folder_path) + interface.run_conversion(nwbfile_path=nwbfile_path, overwrite=True) + + with NWBHDF5IO(nwbfile_path, "r") as io: + nwbfile = io.read() + assert len(nwbfile.acquisition) == 1 # Only one channel has data for this set + events = nwbfile.acquisition["EventsNIDQDigitalChannelXD0"] + assert events.name == "EventsNIDQDigitalChannelXD0" + assert events.timestamps.size == 326 + assert len(nwbfile.devices) == 1 + + data = events.data + # Check that there is one followed by 0 + np.sum(data == 1) == 163 + np.sum(data == 0) == 163 + + +def test_nidq_interface_analog_data(tmp_path): + + nwbfile_path = tmp_path / "nidq_test_analog.nwb" + folder_path = ECEPHY_DATA_PATH / "spikeglx" / "Noise4Sam_g0" + interface = SpikeGLXNIDQInterface(folder_path=folder_path) + interface.run_conversion(nwbfile_path=nwbfile_path, overwrite=True) + + with NWBHDF5IO(nwbfile_path, "r") as io: + nwbfile = io.read() + assert len(nwbfile.acquisition) == 1 # The time series object + time_series = nwbfile.acquisition["TimeSeriesNIDQ"] + assert time_series.name == "TimeSeriesNIDQ" + expected_description = "Analog data from the NIDQ board. Channels are ['XA0' 'XA1' 'XA2' 'XA3' 'XA4' 'XA5' 'XA6' 'XA7'] in that order." + assert time_series.description == expected_description + number_of_samples = time_series.data.shape[0] + assert number_of_samples == 60_864 + number_of_channels = time_series.data.shape[1] + assert number_of_channels == 8 + + assert len(nwbfile.devices) == 1 diff --git a/tests/test_on_data/ecephys/test_spikeglx_converter.py b/tests/test_on_data/ecephys/test_spikeglx_converter.py index 970a815af..93b228053 100644 --- a/tests/test_on_data/ecephys/test_spikeglx_converter.py +++ b/tests/test_on_data/ecephys/test_spikeglx_converter.py @@ -37,16 +37,16 @@ def assertNWBFileStructure(self, nwbfile_path: FilePath, expected_session_start_ assert "ElectricalSeriesAP" in nwbfile.acquisition assert "ElectricalSeriesLF" in nwbfile.acquisition - assert "ElectricalSeriesNIDQ" in nwbfile.acquisition + assert "TimeSeriesNIDQ" in nwbfile.acquisition + assert len(nwbfile.acquisition) == 3 assert "NeuropixelImec0" in nwbfile.devices assert "NIDQBoard" in nwbfile.devices assert len(nwbfile.devices) == 2 - assert "NIDQChannelGroup" in nwbfile.electrode_groups assert "Imec0" in nwbfile.electrode_groups - assert len(nwbfile.electrode_groups) == 2 + assert len(nwbfile.electrode_groups) == 1 def test_single_probe_spikeglx_converter(self): converter = SpikeGLXConverterPipe(folder_path=SPIKEGLX_PATH / "Noise4Sam_g0") @@ -63,14 +63,13 @@ def test_single_probe_spikeglx_converter(self): expected_ecephys_metadata = expected_metadata["Ecephys"] test_ecephys_metadata = test_metadata["Ecephys"] + assert test_ecephys_metadata == expected_ecephys_metadata device_metadata = test_ecephys_metadata.pop("Device") expected_device_metadata = expected_ecephys_metadata.pop("Device") assert device_metadata == expected_device_metadata - assert test_ecephys_metadata == expected_ecephys_metadata - nwbfile_path = self.tmpdir / "test_single_probe_spikeglx_converter.nwb" converter.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) @@ -183,16 +182,6 @@ def test_electrode_table_writing(tmp_path): electrodes_table = nwbfile.electrodes - # Test NIDQ - electrical_series = nwbfile.acquisition["ElectricalSeriesNIDQ"] - nidq_electrodes_table_region = electrical_series.electrodes - region_indices = nidq_electrodes_table_region.data - recording_extractor = converter.data_interface_objects["nidq"].recording_extractor - - saved_channel_names = electrodes_table[region_indices]["channel_name"] - expected_channel_names_nidq = recording_extractor.get_property("channel_name") - np.testing.assert_array_equal(saved_channel_names, expected_channel_names_nidq) - # Test AP electrical_series = nwbfile.acquisition["ElectricalSeriesAP"] ap_electrodes_table_region = electrical_series.electrodes @@ -236,11 +225,3 @@ def test_electrode_table_writing(tmp_path): channel_ids = recording_extractor_lf.get_channel_ids() np.testing.assert_array_equal(channel_ids, expected_channel_names_lf) - - recording_extractor_nidq = NwbRecordingExtractor( - file_path=nwbfile_path, - electrical_series_name="ElectricalSeriesNIDQ", - ) - - channel_ids = recording_extractor_nidq.get_channel_ids() - np.testing.assert_array_equal(channel_ids, expected_channel_names_nidq) diff --git a/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py b/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py index c8c6bad09..081cd172d 100644 --- a/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py +++ b/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py @@ -75,7 +75,6 @@ def assertNWBFileTimesAligned(self, nwbfile_path: Union[str, Path]): # High level groups were written to file assert "BehaviorEvents" in nwbfile.acquisition - assert "ElectricalSeriesNIDQ" in nwbfile.acquisition assert "trials" in nwbfile.intervals # Aligned data was written