From 9d813e722f87aceacfdc1ab51abed91a79a9a7f2 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 18 Nov 2024 09:20:39 -0800 Subject: [PATCH 01/21] fixed bug with ephys starting time --- .../schneider_2024_convert_session.py | 54 +++++++++---------- .../schneider_2024_metadata.yaml | 16 +++--- .../schneider_2024/schneider_2024_notes.md | 12 +++++ 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py index b4e7e7d..a8ddb57 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py +++ b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py @@ -1,10 +1,7 @@ """Primary script to run to convert an entire session for of data using the NWBConverter.""" from pathlib import Path -import datetime -import pytz from zoneinfo import ZoneInfo import shutil -from pprint import pprint import numpy as np from neuroconv.utils import load_dict_from_file, dict_deep_update @@ -32,7 +29,6 @@ def session_to_nwb( video_file_paths = sorted(video_file_paths) if stub_test: output_dir_path = output_dir_path / "nwb_stub" - recording_folder_path = recording_folder_path.with_name(recording_folder_path.name + "_stubbed") output_dir_path.mkdir(parents=True, exist_ok=True) session_id = "sample_session" @@ -55,18 +51,18 @@ def session_to_nwb( conversion_options.update(dict(Behavior=dict())) # Add Video(s) - for i, video_file_path in enumerate(video_file_paths): - metadata_key_name = f"VideoCamera{i+1}" - source_data.update({metadata_key_name: dict(file_paths=[video_file_path], metadata_key_name=metadata_key_name)}) - conversion_options.update({metadata_key_name: dict()}) + # for i, video_file_path in enumerate(video_file_paths): + # metadata_key_name = f"VideoCamera{i+1}" + # source_data.update({metadata_key_name: dict(file_paths=[video_file_path], metadata_key_name=metadata_key_name)}) + # conversion_options.update({metadata_key_name: dict()}) # Add Optogenetic - source_data.update(dict(Optogenetic=dict(file_path=behavior_file_path))) - conversion_options.update(dict(Optogenetic=dict())) + # source_data.update(dict(Optogenetic=dict(file_path=behavior_file_path))) + # conversion_options.update(dict(Optogenetic=dict())) # Add Intrinsic Signal Optical Imaging - source_data.update(dict(ISOI=dict(folder_path=intrinsic_signal_optical_imaging_folder_path))) - conversion_options.update(dict(ISOI=dict())) + # source_data.update(dict(ISOI=dict(folder_path=intrinsic_signal_optical_imaging_folder_path))) + # conversion_options.update(dict(ISOI=dict())) converter = Schneider2024NWBConverter(source_data=source_data) @@ -85,7 +81,7 @@ def session_to_nwb( # Add electrode metadata channel_positions = np.load(sorting_folder_path / "channel_positions.npy") - if stub_test: + if True: # stub_test: SWITCH BACK TO stub_test WHEN ALL CHANNELS ARE PRESENT channel_positions = channel_positions[:1, :] location = metadata["Ecephys"]["ElectrodeGroup"][0]["location"] channel_ids = converter.data_interface_objects["Recording"].recording_extractor.get_channel_ids() @@ -97,34 +93,38 @@ def session_to_nwb( ids=channel_ids, values=[location] * len(channel_ids), ) + converter.data_interface_objects["Recording"].recording_extractor._recording_segments[0].t_start = 0.0 metadata["Ecephys"]["Device"] = editable_metadata["Ecephys"]["Device"] - # Overwrite video metadata - for i, video_file_path in enumerate(video_file_paths): - metadata_key_name = f"VideoCamera{i+1}" - metadata["Behavior"][metadata_key_name] = editable_metadata["Behavior"][metadata_key_name] + # # Overwrite video metadata + # for i, video_file_path in enumerate(video_file_paths): + # metadata_key_name = f"VideoCamera{i+1}" + # metadata["Behavior"][metadata_key_name] = editable_metadata["Behavior"][metadata_key_name] # Run conversion + from time import time + + start = time() converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options) + stop = time() + print(f"Conversion took {stop-start:.2f} seconds") def main(): # Parameters for conversion - data_dir_path = Path("/Volumes/T7/CatalystNeuro/Schneider") + data_dir_path = Path("/Volumes/T7/CatalystNeuro/Schneider/Grant Zempolich Project Data") output_dir_path = Path("/Volumes/T7/CatalystNeuro/Schneider/conversion_nwb") - stub_test = True + stub_test = False if output_dir_path.exists(): shutil.rmtree(output_dir_path, ignore_errors=True) - # Example Session w/ old ephys + new behavior - recording_folder_path = data_dir_path / "Schneider sample Data" / "Raw Ephys" / "m69_2023-10-31_17-24-15_Day1_A1" - sorting_folder_path = ( - data_dir_path / "Schneider sample Data" / "Processed Ephys" / "m69_2023-10-31_17-24-15_Day1_A1" - ) - behavior_file_path = data_dir_path / "NWB_Share" / "Sample behavior data" / "m74_optoSample.mat" - video_folder_path = data_dir_path / "Schneider sample Data" / "Video" / "m69_231031" - intrinsic_signal_optical_imaging_folder_path = data_dir_path / "NWB_Share" / "Sample Intrinsic imaging data" + # Example Session A1 Ephys + Behavior + recording_folder_path = data_dir_path / "A1_EphysFiles" / "m53" / "Day1_A1" + sorting_folder_path = recording_folder_path + behavior_file_path = data_dir_path / "A1_EphysBehavioralFiles" / "raw_m53_231029_001.mat" + video_folder_path = Path("") + intrinsic_signal_optical_imaging_folder_path = Path("") session_to_nwb( recording_folder_path=recording_folder_path, sorting_folder_path=sorting_folder_path, diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_metadata.yaml b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_metadata.yaml index d526a33..ae8d0d9 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_metadata.yaml +++ b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_metadata.yaml @@ -76,14 +76,14 @@ Behavior: description: Position of start of the target zone on a given trial (in raw encoder values corresponding to value read out by quadrature encoder). - name: endZone_ThresholdVector description: Position of ending/exit position of target zone on a given trial (in raw encoder values corresponding to value read out by quadrature encoder). - VideoCamera1: - - name: video_camera_1 - description: Two IR video cameras (AAK CA20 600TVL 2.8MM) are used to monitor the experiments from different angles of interest, allowing for offline analysis of body movements, pupillometry, and other behavioral data if necessary. Camera 1 is a side angle view of the mouse. - unit: Frames - VideoCamera2: - - name: video_camera_2 - description: Two IR video cameras (AAK CA20 600TVL 2.8MM) are used to monitor the experiments from different angles of interest, allowing for offline analysis of body movements, pupillometry, and other behavioral data if necessary. Camera 2 is a zoomed-in view of the pupil of the mouse. - unit: Frames + # VideoCamera1: + # - name: video_camera_1 + # description: Two IR video cameras (AAK CA20 600TVL 2.8MM) are used to monitor the experiments from different angles of interest, allowing for offline analysis of body movements, pupillometry, and other behavioral data if necessary. Camera 1 is a side angle view of the mouse. + # unit: Frames + # VideoCamera2: + # - name: video_camera_2 + # description: Two IR video cameras (AAK CA20 600TVL 2.8MM) are used to monitor the experiments from different angles of interest, allowing for offline analysis of body movements, pupillometry, and other behavioral data if necessary. Camera 2 is a zoomed-in view of the pupil of the mouse. + # unit: Frames Sorting: units_description: Neural spikes will be sorted offline using Kilosort 2.5 and Phy2 software and manually curated to ensure precise spike time acquisition. diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_notes.md b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_notes.md index 1a2030e..eda210e 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_notes.md +++ b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_notes.md @@ -15,6 +15,18 @@ - Need device info for 2p microscope and red light laser - Why is the overlaid image flipped left/right compared to the original? +# Temporal Alignment +- For session A1/m53/Day1 (raw_m53_231029_001.mat), + - ephys data starts at 1025s with duration 2700s + - units table runs from 0-2700 + - behavioral time series (lick and encoder) run from 1187s to 2017s + - events table (toneIN, toneOUT, targetOUT, valve) run from 1191s to 1954s + - valued events table () runs from 2017 to 2164 + - trials table () runs from 1191 to 1993 + --> conclusion: something is wrong with ephys start time +- Want to split data into epochs: Active Behavior, Passive Listening, ??? What is happening post-2164? Before 1187s? + + ## Data Requests - Mice sexes From a87c6bfc3bd2ef8e05fc9660d34accd1161e8251 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 18 Nov 2024 12:04:47 -0800 Subject: [PATCH 02/21] added fix for ambiguous session start time from openephys --- .../schneider_2024_convert_session.py | 17 ++++++++++++----- .../schneider_2024/schneider_2024_metadata.yaml | 2 ++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py index a8ddb57..724655e 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py +++ b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py @@ -3,6 +3,7 @@ from zoneinfo import ZoneInfo import shutil import numpy as np +from datetime import datetime from neuroconv.utils import load_dict_from_file, dict_deep_update from schneider_lab_to_nwb.schneider_2024 import Schneider2024NWBConverter @@ -38,7 +39,7 @@ def session_to_nwb( conversion_options = dict() # Add Recording - stream_name = "Signals CH" # stream_names = ["Signals CH", "Signals AUX"] + stream_name = "Signals CH" source_data.update(dict(Recording=dict(folder_path=recording_folder_path, stream_name=stream_name))) conversion_options.update(dict(Recording=dict(stub_test=stub_test))) @@ -65,16 +66,22 @@ def session_to_nwb( # conversion_options.update(dict(ISOI=dict())) converter = Schneider2024NWBConverter(source_data=source_data) - - # Add datetime to conversion metadata = converter.get_metadata() - EST = ZoneInfo("US/Eastern") - metadata["NWBFile"]["session_start_time"] = metadata["NWBFile"]["session_start_time"].replace(tzinfo=EST) # Update default metadata with the editable in the corresponding yaml file editable_metadata_path = Path(__file__).parent / "schneider_2024_metadata.yaml" editable_metadata = load_dict_from_file(editable_metadata_path) metadata = dict_deep_update(metadata, editable_metadata) + folder_name = ( + source_data["Recording"]["folder_path"].parent.name + "/" + source_data["Recording"]["folder_path"].name + ) + folder_name_to_start_datetime = metadata["Ecephys"].pop("folder_name_to_start_datetime") + if folder_name in folder_name_to_start_datetime.keys(): + metadata["NWBFile"]["session_start_time"] = datetime.fromisoformat(folder_name_to_start_datetime[folder_name]) + + # Add datetime to conversion + EST = ZoneInfo("US/Eastern") + metadata["NWBFile"]["session_start_time"] = metadata["NWBFile"]["session_start_time"].replace(tzinfo=EST) metadata["Subject"]["subject_id"] = "a_subject_id" # Modify here or in the yaml file conversion_options["Sorting"]["units_description"] = metadata["Sorting"]["units_description"] diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_metadata.yaml b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_metadata.yaml index ae8d0d9..61e6239 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_metadata.yaml +++ b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_metadata.yaml @@ -30,6 +30,8 @@ Ecephys: ElectricalSeries: - name: ElectricalSeries description: Recording of AC neural responses in mice performing this behavioral task will utilize dense 128-channel recording probes (Masmanidis Lab). These recording probes span a depth ~1mm, allowing for sampling of all layers of cortex. Electrophysiology data will be recorded using OpenEphys Acquisition Board v2.4 and associated OpenEphys GUI software. + folder_name_to_start_datetime: + m53/Day1_A1: 2023-10-29T16:56:01 Behavior: Module: From 3ac10493b1615f3617b4dc3a91a9924b2d1de68a Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 18 Nov 2024 12:28:33 -0800 Subject: [PATCH 03/21] added epochs --- .../schneider_2024/schneider_2024_behaviorinterface.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_behaviorinterface.py b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_behaviorinterface.py index 63b7b97..45a9ea1 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_behaviorinterface.py +++ b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_behaviorinterface.py @@ -192,6 +192,14 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): trial_array = name_to_trial_array[name] nwbfile.add_trial_column(name=name, description=trials_dict["description"], data=trial_array) + # Add Epochs Table + nwbfile.add_epoch(start_time=trial_start_times[0], stop_time=trial_stop_times[-1], tags=["Active Behavior"]) + nwbfile.add_epoch( + start_time=valued_events_table["timestamp"][0], + stop_time=valued_events_table["timestamp"][-1], + tags=["Passive Listening"], + ) + # Add Devices for device_kwargs in metadata["Behavior"]["Devices"]: device = Device(**device_kwargs) From 7256e3b36e65bfe88fa60639c2d13830d47b6f8e Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 18 Nov 2024 12:40:21 -0800 Subject: [PATCH 04/21] removed nan trials --- .../schneider_2024/schneider_2024_behaviorinterface.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_behaviorinterface.py b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_behaviorinterface.py index 45a9ea1..402fe15 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_behaviorinterface.py +++ b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_behaviorinterface.py @@ -119,10 +119,13 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): trial_start_times = np.array(file["events"]["push"]["time"]).squeeze() trial_stop_times = np.array(file["events"]["push"]["time_end"]).squeeze() + trial_is_nan = np.isnan(trial_start_times) | np.isnan(trial_stop_times) + trial_start_times = trial_start_times[np.logical_not(trial_is_nan)] + trial_stop_times = trial_stop_times[np.logical_not(trial_is_nan)] for trials_dict in metadata["Behavior"]["Trials"]: name = trials_dict["name"] trial_array = np.array(file["events"]["push"][name]).squeeze() - name_to_trial_array[name] = trial_array + name_to_trial_array[name] = trial_array[np.logical_not(trial_is_nan)] # Add Data to NWBFile behavior_module = nwb_helpers.get_module( From 4dbeb5381d193c2d86446aed0291f40a855f0730 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 18 Nov 2024 12:55:29 -0800 Subject: [PATCH 05/21] refactored for optional opto and optional ephys --- .../schneider_2024_convert_session.py | 81 +++++++++++-------- 1 file changed, 49 insertions(+), 32 deletions(-) diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py index 724655e..16f130d 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py +++ b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py @@ -4,22 +4,21 @@ import shutil import numpy as np from datetime import datetime +from typing import Optional from neuroconv.utils import load_dict_from_file, dict_deep_update from schneider_lab_to_nwb.schneider_2024 import Schneider2024NWBConverter def session_to_nwb( - recording_folder_path: str | Path, - sorting_folder_path: str | Path, behavior_file_path: str | Path, video_folder_path: str | Path, intrinsic_signal_optical_imaging_folder_path: str | Path, output_dir_path: str | Path, + ephys_folder_path: Optional[str | Path] = None, + has_opto: bool = False, stub_test: bool = False, ): - recording_folder_path = Path(recording_folder_path) - sorting_folder_path = Path(sorting_folder_path) behavior_file_path = Path(behavior_file_path) video_folder_path = Path(video_folder_path) intrinsic_signal_optical_imaging_folder_path = Path(intrinsic_signal_optical_imaging_folder_path) @@ -31,6 +30,11 @@ def session_to_nwb( if stub_test: output_dir_path = output_dir_path / "nwb_stub" output_dir_path.mkdir(parents=True, exist_ok=True) + if ephys_folder_path is None: + has_ephys = False + else: + has_ephys = True + ephys_folder_path = Path(ephys_folder_path) session_id = "sample_session" nwbfile_path = output_dir_path / f"{session_id}.nwb" @@ -38,14 +42,14 @@ def session_to_nwb( source_data = dict() conversion_options = dict() - # Add Recording - stream_name = "Signals CH" - source_data.update(dict(Recording=dict(folder_path=recording_folder_path, stream_name=stream_name))) - conversion_options.update(dict(Recording=dict(stub_test=stub_test))) + # Add Ephys Recording and Sorting + if has_ephys: + stream_name = "Signals CH" + source_data.update(dict(Recording=dict(folder_path=ephys_folder_path, stream_name=stream_name))) + conversion_options.update(dict(Recording=dict(stub_test=stub_test))) - # Add Sorting - source_data.update(dict(Sorting=dict(folder_path=sorting_folder_path))) - conversion_options.update(dict(Sorting=dict())) + source_data.update(dict(Sorting=dict(folder_path=ephys_folder_path))) + conversion_options.update(dict(Sorting=dict())) # Add Behavior source_data.update(dict(Behavior=dict(file_path=behavior_file_path))) @@ -58,8 +62,9 @@ def session_to_nwb( # conversion_options.update({metadata_key_name: dict()}) # Add Optogenetic - # source_data.update(dict(Optogenetic=dict(file_path=behavior_file_path))) - # conversion_options.update(dict(Optogenetic=dict())) + if has_opto: + source_data.update(dict(Optogenetic=dict(file_path=behavior_file_path))) + conversion_options.update(dict(Optogenetic=dict())) # Add Intrinsic Signal Optical Imaging # source_data.update(dict(ISOI=dict(folder_path=intrinsic_signal_optical_imaging_folder_path))) @@ -87,21 +92,22 @@ def session_to_nwb( conversion_options["Sorting"]["units_description"] = metadata["Sorting"]["units_description"] # Add electrode metadata - channel_positions = np.load(sorting_folder_path / "channel_positions.npy") - if True: # stub_test: SWITCH BACK TO stub_test WHEN ALL CHANNELS ARE PRESENT - channel_positions = channel_positions[:1, :] - location = metadata["Ecephys"]["ElectrodeGroup"][0]["location"] - channel_ids = converter.data_interface_objects["Recording"].recording_extractor.get_channel_ids() - converter.data_interface_objects["Recording"].recording_extractor.set_channel_locations( - channel_ids=channel_ids, locations=channel_positions - ) - converter.data_interface_objects["Recording"].recording_extractor.set_property( - key="brain_area", - ids=channel_ids, - values=[location] * len(channel_ids), - ) - converter.data_interface_objects["Recording"].recording_extractor._recording_segments[0].t_start = 0.0 - metadata["Ecephys"]["Device"] = editable_metadata["Ecephys"]["Device"] + if has_ephys: + channel_positions = np.load(ephys_folder_path / "channel_positions.npy") + if True: # stub_test: SWITCH BACK TO stub_test WHEN ALL CHANNELS ARE PRESENT + channel_positions = channel_positions[:1, :] + location = metadata["Ecephys"]["ElectrodeGroup"][0]["location"] + channel_ids = converter.data_interface_objects["Recording"].recording_extractor.get_channel_ids() + converter.data_interface_objects["Recording"].recording_extractor.set_channel_locations( + channel_ids=channel_ids, locations=channel_positions + ) + converter.data_interface_objects["Recording"].recording_extractor.set_property( + key="brain_area", + ids=channel_ids, + values=[location] * len(channel_ids), + ) + converter.data_interface_objects["Recording"].recording_extractor._recording_segments[0].t_start = 0.0 + metadata["Ecephys"]["Device"] = editable_metadata["Ecephys"]["Device"] # # Overwrite video metadata # for i, video_file_path in enumerate(video_file_paths): @@ -127,14 +133,12 @@ def main(): shutil.rmtree(output_dir_path, ignore_errors=True) # Example Session A1 Ephys + Behavior - recording_folder_path = data_dir_path / "A1_EphysFiles" / "m53" / "Day1_A1" - sorting_folder_path = recording_folder_path + ephys_folder_path = data_dir_path / "A1_EphysFiles" / "m53" / "Day1_A1" behavior_file_path = data_dir_path / "A1_EphysBehavioralFiles" / "raw_m53_231029_001.mat" video_folder_path = Path("") intrinsic_signal_optical_imaging_folder_path = Path("") session_to_nwb( - recording_folder_path=recording_folder_path, - sorting_folder_path=sorting_folder_path, + ephys_folder_path=ephys_folder_path, behavior_file_path=behavior_file_path, video_folder_path=video_folder_path, intrinsic_signal_optical_imaging_folder_path=intrinsic_signal_optical_imaging_folder_path, @@ -142,6 +146,19 @@ def main(): stub_test=stub_test, ) + # # Example Session A1 Ogen + Behavior + # behavior_file_path = data_dir_path / "A1_OptoBehavioralFiles" / "raw_m53_231013_001.mat" + # video_folder_path = Path("") + # intrinsic_signal_optical_imaging_folder_path = Path("") + # session_to_nwb( + # ephys_folder_path=recording_folder_path, + # behavior_file_path=behavior_file_path, + # video_folder_path=video_folder_path, + # intrinsic_signal_optical_imaging_folder_path=intrinsic_signal_optical_imaging_folder_path, + # output_dir_path=output_dir_path, + # stub_test=stub_test, + # ) + if __name__ == "__main__": main() From b8093d8d32a2d31b431e78f119a01d23895d6058 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 18 Nov 2024 12:56:17 -0800 Subject: [PATCH 06/21] refactored for optional opto and optional ephys --- .../schneider_2024_convert_session.py | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py index 16f130d..0a89c6a 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py +++ b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py @@ -115,12 +115,7 @@ def session_to_nwb( # metadata["Behavior"][metadata_key_name] = editable_metadata["Behavior"][metadata_key_name] # Run conversion - from time import time - - start = time() converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options) - stop = time() - print(f"Conversion took {stop-start:.2f} seconds") def main(): @@ -146,18 +141,18 @@ def main(): stub_test=stub_test, ) - # # Example Session A1 Ogen + Behavior - # behavior_file_path = data_dir_path / "A1_OptoBehavioralFiles" / "raw_m53_231013_001.mat" - # video_folder_path = Path("") - # intrinsic_signal_optical_imaging_folder_path = Path("") - # session_to_nwb( - # ephys_folder_path=recording_folder_path, - # behavior_file_path=behavior_file_path, - # video_folder_path=video_folder_path, - # intrinsic_signal_optical_imaging_folder_path=intrinsic_signal_optical_imaging_folder_path, - # output_dir_path=output_dir_path, - # stub_test=stub_test, - # ) + # Example Session A1 Ogen + Behavior + behavior_file_path = data_dir_path / "A1_OptoBehavioralFiles" / "raw_m53_231013_001.mat" + video_folder_path = Path("") + intrinsic_signal_optical_imaging_folder_path = Path("") + session_to_nwb( + behavior_file_path=behavior_file_path, + video_folder_path=video_folder_path, + intrinsic_signal_optical_imaging_folder_path=intrinsic_signal_optical_imaging_folder_path, + output_dir_path=output_dir_path, + has_opto=True, + stub_test=stub_test, + ) if __name__ == "__main__": From e6d4fabadfb335cb6824a3d19f9637612220c476 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 18 Nov 2024 13:19:55 -0800 Subject: [PATCH 07/21] added example opto session --- .../schneider_2024_convert_session.py | 29 ++++++++++++------- .../schneider_2024/schneider_2024_notes.md | 1 + 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py index 0a89c6a..31c6f0f 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py +++ b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py @@ -36,9 +36,6 @@ def session_to_nwb( has_ephys = True ephys_folder_path = Path(ephys_folder_path) - session_id = "sample_session" - nwbfile_path = output_dir_path / f"{session_id}.nwb" - source_data = dict() conversion_options = dict() @@ -77,19 +74,25 @@ def session_to_nwb( editable_metadata_path = Path(__file__).parent / "schneider_2024_metadata.yaml" editable_metadata = load_dict_from_file(editable_metadata_path) metadata = dict_deep_update(metadata, editable_metadata) - folder_name = ( - source_data["Recording"]["folder_path"].parent.name + "/" + source_data["Recording"]["folder_path"].name - ) - folder_name_to_start_datetime = metadata["Ecephys"].pop("folder_name_to_start_datetime") - if folder_name in folder_name_to_start_datetime.keys(): - metadata["NWBFile"]["session_start_time"] = datetime.fromisoformat(folder_name_to_start_datetime[folder_name]) + if has_ephys: + folder_name = ( + source_data["Recording"]["folder_path"].parent.name + "/" + source_data["Recording"]["folder_path"].name + ) + folder_name_to_start_datetime = metadata["Ecephys"].pop("folder_name_to_start_datetime") + if folder_name in folder_name_to_start_datetime.keys(): + metadata["NWBFile"]["session_start_time"] = datetime.fromisoformat( + folder_name_to_start_datetime[folder_name] + ) + else: + metadata["NWBFile"]["session_start_time"] = datetime.strptime(behavior_file_path.name.split("_")[2], "%y%m%d") # Add datetime to conversion EST = ZoneInfo("US/Eastern") metadata["NWBFile"]["session_start_time"] = metadata["NWBFile"]["session_start_time"].replace(tzinfo=EST) metadata["Subject"]["subject_id"] = "a_subject_id" # Modify here or in the yaml file - conversion_options["Sorting"]["units_description"] = metadata["Sorting"]["units_description"] + if has_ephys: + conversion_options["Sorting"]["units_description"] = metadata["Sorting"]["units_description"] # Add electrode metadata if has_ephys: @@ -114,6 +117,12 @@ def session_to_nwb( # metadata_key_name = f"VideoCamera{i+1}" # metadata["Behavior"][metadata_key_name] = editable_metadata["Behavior"][metadata_key_name] + subject_id = behavior_file_path.name.split("_")[1] + session_id = behavior_file_path.name.split("_")[2] + nwbfile_path = output_dir_path / f"sub-{subject_id}_ses-{session_id}.nwb" + metadata["NWBFile"]["session_id"] = session_id + metadata["Subject"]["subject_id"] = subject_id + # Run conversion converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options) diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_notes.md b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_notes.md index eda210e..6e029cb 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_notes.md +++ b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_notes.md @@ -25,6 +25,7 @@ - trials table () runs from 1191 to 1993 --> conclusion: something is wrong with ephys start time - Want to split data into epochs: Active Behavior, Passive Listening, ??? What is happening post-2164? Before 1187s? +- For opto sessions, what is the session start time? Ex. What is file['metadata']['session_beginning'] (=129765.7728241)? From d9291b66212e066aa0fd7ab8455c377619831025 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 18 Nov 2024 13:28:20 -0800 Subject: [PATCH 08/21] renamed schneider --> zempolich --- .../schneider_2024/__init__.py | 4 - .../schneider_2024_requirements.txt | 3 - .../zempolich_2024/__init__.py | 4 + .../schneider_2024_convert_session.py | 4 +- .../zempolich_2024_behaviorinterface.py} | 2 +- .../zempolich_2024_convert_all_sessions.py} | 2 +- .../zempolich_2024_convert_session.py | 168 ++++++++++++++++++ ...024_intrinsic_signal_imaging_interface.py} | 2 +- .../zempolich_2024_metadata.yaml} | 0 .../zempolich_2024_notes.md} | 0 .../zempolich_2024_nwbconverter.py} | 16 +- .../zempolich_2024_optogeneticinterface.py} | 2 +- 12 files changed, 186 insertions(+), 21 deletions(-) delete mode 100644 src/schneider_lab_to_nwb/schneider_2024/__init__.py delete mode 100644 src/schneider_lab_to_nwb/schneider_2024/schneider_2024_requirements.txt create mode 100644 src/schneider_lab_to_nwb/zempolich_2024/__init__.py rename src/schneider_lab_to_nwb/{schneider_2024 => zempolich_2024}/schneider_2024_convert_session.py (98%) rename src/schneider_lab_to_nwb/{schneider_2024/schneider_2024_behaviorinterface.py => zempolich_2024/zempolich_2024_behaviorinterface.py} (99%) rename src/schneider_lab_to_nwb/{schneider_2024/schneider_2024_convert_all_sessions.py => zempolich_2024/zempolich_2024_convert_all_sessions.py} (98%) create mode 100644 src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py rename src/schneider_lab_to_nwb/{schneider_2024/schneider_2024_intrinsic_signal_imaging_interface.py => zempolich_2024/zempolich_2024_intrinsic_signal_imaging_interface.py} (98%) rename src/schneider_lab_to_nwb/{schneider_2024/schneider_2024_metadata.yaml => zempolich_2024/zempolich_2024_metadata.yaml} (100%) rename src/schneider_lab_to_nwb/{schneider_2024/schneider_2024_notes.md => zempolich_2024/zempolich_2024_notes.md} (100%) rename src/schneider_lab_to_nwb/{schneider_2024/schneider_2024_nwbconverter.py => zempolich_2024/zempolich_2024_nwbconverter.py} (58%) rename src/schneider_lab_to_nwb/{schneider_2024/schneider_2024_optogeneticinterface.py => zempolich_2024/zempolich_2024_optogeneticinterface.py} (97%) diff --git a/src/schneider_lab_to_nwb/schneider_2024/__init__.py b/src/schneider_lab_to_nwb/schneider_2024/__init__.py deleted file mode 100644 index 11f8f68..0000000 --- a/src/schneider_lab_to_nwb/schneider_2024/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .schneider_2024_behaviorinterface import Schneider2024BehaviorInterface -from .schneider_2024_optogeneticinterface import Schneider2024OptogeneticInterface -from .schneider_2024_intrinsic_signal_imaging_interface import Schneider2024IntrinsicSignalOpticalImagingInterface -from .schneider_2024_nwbconverter import Schneider2024NWBConverter diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_requirements.txt b/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_requirements.txt deleted file mode 100644 index 458b8a2..0000000 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -nwb-conversion-tools==0.11.1 # Example of specific pinned dependecy -some-extra-package==1.11.3 # Example of another extra package that's necessary for the current conversion -roiextractors @ git+https://github.com/catalystneuro/roiextractors.git@8db5f9cb3a7ee5efee49b7fd0b694c7a8105519a # Github pinned dependency diff --git a/src/schneider_lab_to_nwb/zempolich_2024/__init__.py b/src/schneider_lab_to_nwb/zempolich_2024/__init__.py new file mode 100644 index 0000000..760505c --- /dev/null +++ b/src/schneider_lab_to_nwb/zempolich_2024/__init__.py @@ -0,0 +1,4 @@ +from .zempolich_2024_behaviorinterface import Zempolich2024BehaviorInterface +from .zempolich_2024_optogeneticinterface import Zempolich2024OptogeneticInterface +from .zempolich_2024_intrinsic_signal_imaging_interface import Zempolich2024IntrinsicSignalOpticalImagingInterface +from .zempolich_2024_nwbconverter import Zempolich2024NWBConverter diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py b/src/schneider_lab_to_nwb/zempolich_2024/schneider_2024_convert_session.py similarity index 98% rename from src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py rename to src/schneider_lab_to_nwb/zempolich_2024/schneider_2024_convert_session.py index 31c6f0f..f44d076 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_session.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/schneider_2024_convert_session.py @@ -7,7 +7,7 @@ from typing import Optional from neuroconv.utils import load_dict_from_file, dict_deep_update -from schneider_lab_to_nwb.schneider_2024 import Schneider2024NWBConverter +from schneider_lab_to_nwb.zempolich_2024 import Zempolich2024NWBConverter def session_to_nwb( @@ -67,7 +67,7 @@ def session_to_nwb( # source_data.update(dict(ISOI=dict(folder_path=intrinsic_signal_optical_imaging_folder_path))) # conversion_options.update(dict(ISOI=dict())) - converter = Schneider2024NWBConverter(source_data=source_data) + converter = Zempolich2024NWBConverter(source_data=source_data) metadata = converter.get_metadata() # Update default metadata with the editable in the corresponding yaml file diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_behaviorinterface.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py similarity index 99% rename from src/schneider_lab_to_nwb/schneider_2024/schneider_2024_behaviorinterface.py rename to src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py index 402fe15..24c3700 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_behaviorinterface.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py @@ -13,7 +13,7 @@ from neuroconv.tools import nwb_helpers -class Schneider2024BehaviorInterface(BaseDataInterface): +class Zempolich2024BehaviorInterface(BaseDataInterface): """Behavior interface for schneider_2024 conversion""" keywords = ("behavior",) diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_all_sessions.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_all_sessions.py similarity index 98% rename from src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_all_sessions.py rename to src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_all_sessions.py index 8cb5833..f256fa9 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_convert_all_sessions.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_all_sessions.py @@ -5,7 +5,7 @@ import traceback from tqdm import tqdm -from .schneider_2024_convert_session import session_to_nwb +from .zempolich_2024_convert_session import session_to_nwb def dataset_to_nwb( diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py new file mode 100644 index 0000000..c61c964 --- /dev/null +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py @@ -0,0 +1,168 @@ +"""Primary script to run to convert an entire session for of data using the NWBConverter.""" +from pathlib import Path +from zoneinfo import ZoneInfo +import shutil +import numpy as np +from datetime import datetime +from typing import Optional + +from neuroconv.utils import load_dict_from_file, dict_deep_update +from schneider_lab_to_nwb.zempolich_2024 import Zempolich2024NWBConverter + + +def session_to_nwb( + behavior_file_path: str | Path, + video_folder_path: str | Path, + intrinsic_signal_optical_imaging_folder_path: str | Path, + output_dir_path: str | Path, + ephys_folder_path: Optional[str | Path] = None, + has_opto: bool = False, + stub_test: bool = False, +): + behavior_file_path = Path(behavior_file_path) + video_folder_path = Path(video_folder_path) + intrinsic_signal_optical_imaging_folder_path = Path(intrinsic_signal_optical_imaging_folder_path) + output_dir_path = Path(output_dir_path) + video_file_paths = [ + file_path for file_path in video_folder_path.glob("*.mp4") if not file_path.name.startswith("._") + ] + video_file_paths = sorted(video_file_paths) + if stub_test: + output_dir_path = output_dir_path / "nwb_stub" + output_dir_path.mkdir(parents=True, exist_ok=True) + if ephys_folder_path is None: + has_ephys = False + else: + has_ephys = True + ephys_folder_path = Path(ephys_folder_path) + + source_data = dict() + conversion_options = dict() + + # Add Ephys Recording and Sorting + if has_ephys: + stream_name = "Signals CH" + source_data.update(dict(Recording=dict(folder_path=ephys_folder_path, stream_name=stream_name))) + conversion_options.update(dict(Recording=dict(stub_test=stub_test))) + + source_data.update(dict(Sorting=dict(folder_path=ephys_folder_path))) + conversion_options.update(dict(Sorting=dict())) + + # Add Behavior + source_data.update(dict(Behavior=dict(file_path=behavior_file_path))) + conversion_options.update(dict(Behavior=dict())) + + # Add Video(s) + # for i, video_file_path in enumerate(video_file_paths): + # metadata_key_name = f"VideoCamera{i+1}" + # source_data.update({metadata_key_name: dict(file_paths=[video_file_path], metadata_key_name=metadata_key_name)}) + # conversion_options.update({metadata_key_name: dict()}) + + # Add Optogenetic + if has_opto: + source_data.update(dict(Optogenetic=dict(file_path=behavior_file_path))) + conversion_options.update(dict(Optogenetic=dict())) + + # Add Intrinsic Signal Optical Imaging + # source_data.update(dict(ISOI=dict(folder_path=intrinsic_signal_optical_imaging_folder_path))) + # conversion_options.update(dict(ISOI=dict())) + + converter = Zempolich2024NWBConverter(source_data=source_data) + metadata = converter.get_metadata() + + # Update default metadata with the editable in the corresponding yaml file + editable_metadata_path = Path(__file__).parent / "zempolich_2024_metadata.yaml" + editable_metadata = load_dict_from_file(editable_metadata_path) + metadata = dict_deep_update(metadata, editable_metadata) + if has_ephys: + folder_name = ( + source_data["Recording"]["folder_path"].parent.name + "/" + source_data["Recording"]["folder_path"].name + ) + folder_name_to_start_datetime = metadata["Ecephys"].pop("folder_name_to_start_datetime") + if folder_name in folder_name_to_start_datetime.keys(): + metadata["NWBFile"]["session_start_time"] = datetime.fromisoformat( + folder_name_to_start_datetime[folder_name] + ) + else: + metadata["NWBFile"]["session_start_time"] = datetime.strptime(behavior_file_path.name.split("_")[2], "%y%m%d") + + # Add datetime to conversion + EST = ZoneInfo("US/Eastern") + metadata["NWBFile"]["session_start_time"] = metadata["NWBFile"]["session_start_time"].replace(tzinfo=EST) + + metadata["Subject"]["subject_id"] = "a_subject_id" # Modify here or in the yaml file + if has_ephys: + conversion_options["Sorting"]["units_description"] = metadata["Sorting"]["units_description"] + + # Add electrode metadata + if has_ephys: + channel_positions = np.load(ephys_folder_path / "channel_positions.npy") + if True: # stub_test: SWITCH BACK TO stub_test WHEN ALL CHANNELS ARE PRESENT + channel_positions = channel_positions[:1, :] + location = metadata["Ecephys"]["ElectrodeGroup"][0]["location"] + channel_ids = converter.data_interface_objects["Recording"].recording_extractor.get_channel_ids() + converter.data_interface_objects["Recording"].recording_extractor.set_channel_locations( + channel_ids=channel_ids, locations=channel_positions + ) + converter.data_interface_objects["Recording"].recording_extractor.set_property( + key="brain_area", + ids=channel_ids, + values=[location] * len(channel_ids), + ) + converter.data_interface_objects["Recording"].recording_extractor._recording_segments[0].t_start = 0.0 + metadata["Ecephys"]["Device"] = editable_metadata["Ecephys"]["Device"] + + # # Overwrite video metadata + # for i, video_file_path in enumerate(video_file_paths): + # metadata_key_name = f"VideoCamera{i+1}" + # metadata["Behavior"][metadata_key_name] = editable_metadata["Behavior"][metadata_key_name] + + subject_id = behavior_file_path.name.split("_")[1] + session_id = behavior_file_path.name.split("_")[2] + nwbfile_path = output_dir_path / f"sub-{subject_id}_ses-{session_id}.nwb" + metadata["NWBFile"]["session_id"] = session_id + metadata["Subject"]["subject_id"] = subject_id + + # Run conversion + converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options) + + +def main(): + # Parameters for conversion + data_dir_path = Path("/Volumes/T7/CatalystNeuro/Schneider/Grant Zempolich Project Data") + output_dir_path = Path("/Volumes/T7/CatalystNeuro/Schneider/conversion_nwb") + stub_test = False + + if output_dir_path.exists(): + shutil.rmtree(output_dir_path, ignore_errors=True) + + # Example Session A1 Ephys + Behavior + ephys_folder_path = data_dir_path / "A1_EphysFiles" / "m53" / "Day1_A1" + behavior_file_path = data_dir_path / "A1_EphysBehavioralFiles" / "raw_m53_231029_001.mat" + video_folder_path = Path("") + intrinsic_signal_optical_imaging_folder_path = Path("") + session_to_nwb( + ephys_folder_path=ephys_folder_path, + behavior_file_path=behavior_file_path, + video_folder_path=video_folder_path, + intrinsic_signal_optical_imaging_folder_path=intrinsic_signal_optical_imaging_folder_path, + output_dir_path=output_dir_path, + stub_test=stub_test, + ) + + # Example Session A1 Ogen + Behavior + behavior_file_path = data_dir_path / "A1_OptoBehavioralFiles" / "raw_m53_231013_001.mat" + video_folder_path = Path("") + intrinsic_signal_optical_imaging_folder_path = Path("") + session_to_nwb( + behavior_file_path=behavior_file_path, + video_folder_path=video_folder_path, + intrinsic_signal_optical_imaging_folder_path=intrinsic_signal_optical_imaging_folder_path, + output_dir_path=output_dir_path, + has_opto=True, + stub_test=stub_test, + ) + + +if __name__ == "__main__": + main() diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_intrinsic_signal_imaging_interface.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_intrinsic_signal_imaging_interface.py similarity index 98% rename from src/schneider_lab_to_nwb/schneider_2024/schneider_2024_intrinsic_signal_imaging_interface.py rename to src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_intrinsic_signal_imaging_interface.py index ee6341f..7017f9c 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_intrinsic_signal_imaging_interface.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_intrinsic_signal_imaging_interface.py @@ -12,7 +12,7 @@ from neuroconv.tools import nwb_helpers -class Schneider2024IntrinsicSignalOpticalImagingInterface(BaseDataInterface): +class Zempolich2024IntrinsicSignalOpticalImagingInterface(BaseDataInterface): """Intrinsic signal optical imaging interface for schneider_2024 conversion""" keywords = ("intrinsic signal optical imaging",) diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_metadata.yaml b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml similarity index 100% rename from src/schneider_lab_to_nwb/schneider_2024/schneider_2024_metadata.yaml rename to src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_notes.md b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md similarity index 100% rename from src/schneider_lab_to_nwb/schneider_2024/schneider_2024_notes.md rename to src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_nwbconverter.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_nwbconverter.py similarity index 58% rename from src/schneider_lab_to_nwb/schneider_2024/schneider_2024_nwbconverter.py rename to src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_nwbconverter.py index e51d6e5..b34c690 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_nwbconverter.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_nwbconverter.py @@ -7,22 +7,22 @@ ) from neuroconv.basedatainterface import BaseDataInterface -from schneider_lab_to_nwb.schneider_2024 import ( - Schneider2024BehaviorInterface, - Schneider2024OptogeneticInterface, - Schneider2024IntrinsicSignalOpticalImagingInterface, +from schneider_lab_to_nwb.zempolich_2024 import ( + Zempolich2024BehaviorInterface, + Zempolich2024OptogeneticInterface, + Zempolich2024IntrinsicSignalOpticalImagingInterface, ) -class Schneider2024NWBConverter(NWBConverter): +class Zempolich2024NWBConverter(NWBConverter): """Primary conversion class for my extracellular electrophysiology dataset.""" data_interface_classes = dict( Recording=OpenEphysRecordingInterface, Sorting=PhySortingInterface, - Behavior=Schneider2024BehaviorInterface, + Behavior=Zempolich2024BehaviorInterface, VideoCamera1=VideoInterface, VideoCamera2=VideoInterface, - Optogenetic=Schneider2024OptogeneticInterface, - ISOI=Schneider2024IntrinsicSignalOpticalImagingInterface, + Optogenetic=Zempolich2024OptogeneticInterface, + ISOI=Zempolich2024IntrinsicSignalOpticalImagingInterface, ) diff --git a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_optogeneticinterface.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_optogeneticinterface.py similarity index 97% rename from src/schneider_lab_to_nwb/schneider_2024/schneider_2024_optogeneticinterface.py rename to src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_optogeneticinterface.py index 46b2930..f194abf 100644 --- a/src/schneider_lab_to_nwb/schneider_2024/schneider_2024_optogeneticinterface.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_optogeneticinterface.py @@ -10,7 +10,7 @@ from neuroconv.utils import DeepDict -class Schneider2024OptogeneticInterface(BaseDataInterface): +class Zempolich2024OptogeneticInterface(BaseDataInterface): """Optogenetic interface for schneider_2024 conversion""" keywords = ["optogenetics"] From 5da7b09f37845349814a38edd2c59defd7f7c418 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 18 Nov 2024 15:32:23 -0800 Subject: [PATCH 09/21] refactored ephys to its own interface --- .../zempolich_2024/__init__.py | 1 + .../zempolich_2024_convert_session.py | 56 ++++++++----------- .../zempolich_2024_nwbconverter.py | 4 +- 3 files changed, 25 insertions(+), 36 deletions(-) diff --git a/src/schneider_lab_to_nwb/zempolich_2024/__init__.py b/src/schneider_lab_to_nwb/zempolich_2024/__init__.py index 760505c..441f49d 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/__init__.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/__init__.py @@ -1,4 +1,5 @@ from .zempolich_2024_behaviorinterface import Zempolich2024BehaviorInterface from .zempolich_2024_optogeneticinterface import Zempolich2024OptogeneticInterface from .zempolich_2024_intrinsic_signal_imaging_interface import Zempolich2024IntrinsicSignalOpticalImagingInterface +from .zempolich_2024_open_ephys_recording_interface import Zempolich2024OpenEphysRecordingInterface from .zempolich_2024_nwbconverter import Zempolich2024NWBConverter diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py index c61c964..28277df 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py @@ -2,7 +2,6 @@ from pathlib import Path from zoneinfo import ZoneInfo import shutil -import numpy as np from datetime import datetime from typing import Optional @@ -74,44 +73,14 @@ def session_to_nwb( editable_metadata_path = Path(__file__).parent / "zempolich_2024_metadata.yaml" editable_metadata = load_dict_from_file(editable_metadata_path) metadata = dict_deep_update(metadata, editable_metadata) - if has_ephys: - folder_name = ( - source_data["Recording"]["folder_path"].parent.name + "/" + source_data["Recording"]["folder_path"].name - ) - folder_name_to_start_datetime = metadata["Ecephys"].pop("folder_name_to_start_datetime") - if folder_name in folder_name_to_start_datetime.keys(): - metadata["NWBFile"]["session_start_time"] = datetime.fromisoformat( - folder_name_to_start_datetime[folder_name] - ) - else: - metadata["NWBFile"]["session_start_time"] = datetime.strptime(behavior_file_path.name.split("_")[2], "%y%m%d") - # Add datetime to conversion - EST = ZoneInfo("US/Eastern") - metadata["NWBFile"]["session_start_time"] = metadata["NWBFile"]["session_start_time"].replace(tzinfo=EST) + add_session_start_time_to_metadata( + behavior_file_path=behavior_file_path, ephys_folder_path=ephys_folder_path, metadata=metadata + ) - metadata["Subject"]["subject_id"] = "a_subject_id" # Modify here or in the yaml file if has_ephys: conversion_options["Sorting"]["units_description"] = metadata["Sorting"]["units_description"] - # Add electrode metadata - if has_ephys: - channel_positions = np.load(ephys_folder_path / "channel_positions.npy") - if True: # stub_test: SWITCH BACK TO stub_test WHEN ALL CHANNELS ARE PRESENT - channel_positions = channel_positions[:1, :] - location = metadata["Ecephys"]["ElectrodeGroup"][0]["location"] - channel_ids = converter.data_interface_objects["Recording"].recording_extractor.get_channel_ids() - converter.data_interface_objects["Recording"].recording_extractor.set_channel_locations( - channel_ids=channel_ids, locations=channel_positions - ) - converter.data_interface_objects["Recording"].recording_extractor.set_property( - key="brain_area", - ids=channel_ids, - values=[location] * len(channel_ids), - ) - converter.data_interface_objects["Recording"].recording_extractor._recording_segments[0].t_start = 0.0 - metadata["Ecephys"]["Device"] = editable_metadata["Ecephys"]["Device"] - # # Overwrite video metadata # for i, video_file_path in enumerate(video_file_paths): # metadata_key_name = f"VideoCamera{i+1}" @@ -127,6 +96,25 @@ def session_to_nwb( converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options) +def add_session_start_time_to_metadata( + behavior_file_path: str | Path, + ephys_folder_path: Optional[str | Path], + metadata: dict, +): + if ephys_folder_path is not None: + folder_name = ephys_folder_path.parent.name + "/" + ephys_folder_path.name + folder_name_to_start_datetime = metadata["Ecephys"].pop("folder_name_to_start_datetime") + if folder_name in folder_name_to_start_datetime.keys(): + session_start_time = datetime.fromisoformat(folder_name_to_start_datetime[folder_name]) + else: + session_start_time = metadata["NWBFile"]["session_start_time"] + else: + session_start_time = datetime.strptime(behavior_file_path.name.split("_")[2], "%y%m%d") + + EST = ZoneInfo("US/Eastern") + metadata["NWBFile"]["session_start_time"] = session_start_time.replace(tzinfo=EST) + + def main(): # Parameters for conversion data_dir_path = Path("/Volumes/T7/CatalystNeuro/Schneider/Grant Zempolich Project Data") diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_nwbconverter.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_nwbconverter.py index b34c690..7a67805 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_nwbconverter.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_nwbconverter.py @@ -1,13 +1,13 @@ """Primary NWBConverter class for this dataset.""" from neuroconv import NWBConverter from neuroconv.datainterfaces import ( - OpenEphysRecordingInterface, PhySortingInterface, VideoInterface, ) from neuroconv.basedatainterface import BaseDataInterface from schneider_lab_to_nwb.zempolich_2024 import ( + Zempolich2024OpenEphysRecordingInterface, Zempolich2024BehaviorInterface, Zempolich2024OptogeneticInterface, Zempolich2024IntrinsicSignalOpticalImagingInterface, @@ -18,7 +18,7 @@ class Zempolich2024NWBConverter(NWBConverter): """Primary conversion class for my extracellular electrophysiology dataset.""" data_interface_classes = dict( - Recording=OpenEphysRecordingInterface, + Recording=Zempolich2024OpenEphysRecordingInterface, Sorting=PhySortingInterface, Behavior=Zempolich2024BehaviorInterface, VideoCamera1=VideoInterface, From 4bb588436792fa497d6559232d017dfcf6d148e8 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 18 Nov 2024 15:32:53 -0800 Subject: [PATCH 10/21] refactored ephys to its own interface --- ...ich_2024_open_ephys_recording_interface.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_open_ephys_recording_interface.py diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_open_ephys_recording_interface.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_open_ephys_recording_interface.py new file mode 100644 index 0000000..c45598d --- /dev/null +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_open_ephys_recording_interface.py @@ -0,0 +1,36 @@ +"""Primary class for converting SpikeGadgets Ephys Recordings.""" +from pynwb.file import NWBFile +from pathlib import Path +from xml.etree import ElementTree +from pydantic import FilePath +import copy +from collections import Counter +import numpy as np + +from neuroconv.datainterfaces import OpenEphysLegacyRecordingInterface +from neuroconv.utils import DeepDict +from spikeinterface.extractors import OpenEphysLegacyRecordingExtractor + + +class Zempolich2024OpenEphysRecordingInterface(OpenEphysLegacyRecordingInterface): + """OpenEphys RecordingInterface for zempolich_2024 conversion.""" + + Extractor = OpenEphysLegacyRecordingExtractor + + def get_metadata(self) -> dict: + metadata = super().get_metadata() + metadata["Ecephys"]["Device"] = [] # remove default device + return metadata + + def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, **conversion_options): + folder_path = self.source_data["folder_path"] + channel_positions = np.load(folder_path / "channel_positions.npy") + if True: # TODO: Replace with `if stub_test:` once all channels are present in the data + channel_positions = channel_positions[:1, :] + location = metadata["Ecephys"]["ElectrodeGroup"][0]["location"] + channel_ids = self.recording_extractor.get_channel_ids() + self.recording_extractor.set_channel_locations(channel_ids=channel_ids, locations=channel_positions) + self.recording_extractor.set_property(key="brain_area", ids=channel_ids, values=[location] * len(channel_ids)) + self.recording_extractor._recording_segments[0].t_start = 0.0 + + super().add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, **conversion_options) From 89d04ce9cb0a69a8df7bb8fae00f77132495fe54 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 18 Nov 2024 15:34:00 -0800 Subject: [PATCH 11/21] removed unused imports --- .../zempolich_2024_open_ephys_recording_interface.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_open_ephys_recording_interface.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_open_ephys_recording_interface.py index c45598d..b034ab9 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_open_ephys_recording_interface.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_open_ephys_recording_interface.py @@ -1,14 +1,8 @@ """Primary class for converting SpikeGadgets Ephys Recordings.""" from pynwb.file import NWBFile -from pathlib import Path -from xml.etree import ElementTree -from pydantic import FilePath -import copy -from collections import Counter import numpy as np from neuroconv.datainterfaces import OpenEphysLegacyRecordingInterface -from neuroconv.utils import DeepDict from spikeinterface.extractors import OpenEphysLegacyRecordingExtractor From eb845b70aa497a633ae9eac74598d8b840c85761 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 18 Nov 2024 16:00:26 -0800 Subject: [PATCH 12/21] normalized opto behavior timestamps --- .../zempolich_2024_behaviorinterface.py | 12 +++++++++++- .../zempolich_2024/zempolich_2024_convert_session.py | 1 + .../zempolich_2024/zempolich_2024_notes.md | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py index 24c3700..801067b 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py @@ -89,14 +89,17 @@ def get_metadata_schema(self) -> dict: } return metadata_schema - def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): + def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, normalize_timestamps: bool = False): # Read Data file_path = self.source_data["file_path"] file = read_mat(file_path) behavioral_time_series, name_to_times, name_to_values, name_to_trial_array = [], dict(), dict(), dict() + starting_timestamp = file["continuous"][metadata["Behavior"]["TimeSeries"][0]["name"]]["time"][0] for time_series_dict in metadata["Behavior"]["TimeSeries"]: name = time_series_dict["name"] timestamps = np.array(file["continuous"][name]["time"]).squeeze() + if normalize_timestamps: + timestamps = timestamps - starting_timestamp data = np.array(file["continuous"][name]["value"]).squeeze() time_series = TimeSeries( name=name, @@ -109,10 +112,14 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): for event_dict in metadata["Behavior"]["Events"]: name = event_dict["name"] times = np.array(file["events"][name]["time"]).squeeze() + if normalize_timestamps: + times = times - starting_timestamp name_to_times[name] = times for event_dict in metadata["Behavior"]["ValuedEvents"]: name = event_dict["name"] times = np.array(file["events"][name]["time"]).squeeze() + if normalize_timestamps: + times = times - starting_timestamp values = np.array(file["events"][name]["value"]).squeeze() name_to_times[name] = times name_to_values[name] = values @@ -122,6 +129,9 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): trial_is_nan = np.isnan(trial_start_times) | np.isnan(trial_stop_times) trial_start_times = trial_start_times[np.logical_not(trial_is_nan)] trial_stop_times = trial_stop_times[np.logical_not(trial_is_nan)] + if normalize_timestamps: + trial_start_times = trial_start_times - starting_timestamp + trial_stop_times = trial_stop_times - starting_timestamp for trials_dict in metadata["Behavior"]["Trials"]: name = trials_dict["name"] trial_array = np.array(file["events"]["push"][name]).squeeze() diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py index 28277df..f9ed1be 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py @@ -61,6 +61,7 @@ def session_to_nwb( if has_opto: source_data.update(dict(Optogenetic=dict(file_path=behavior_file_path))) conversion_options.update(dict(Optogenetic=dict())) + conversion_options["Behavior"]["normalize_timestamps"] = True # Add Intrinsic Signal Optical Imaging # source_data.update(dict(ISOI=dict(folder_path=intrinsic_signal_optical_imaging_folder_path))) diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md index 6e029cb..b91c3a6 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md @@ -26,6 +26,7 @@ --> conclusion: something is wrong with ephys start time - Want to split data into epochs: Active Behavior, Passive Listening, ??? What is happening post-2164? Before 1187s? - For opto sessions, what is the session start time? Ex. What is file['metadata']['session_beginning'] (=129765.7728241)? +- Looks like opto sessions are not temporally aligned (no concurrent ephys and timestamp start at large numbers (142697.1119976)) From d0d8db5638cc549d43d728e3c8656c6321d5c163 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 19 Nov 2024 10:16:36 -0800 Subject: [PATCH 13/21] skip missing events --- .../zempolich_2024_behaviorinterface.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py index 801067b..b6b9235 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py @@ -176,6 +176,8 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, normalize_timestamps: ) for event_dict in metadata["Behavior"]["Events"]: event_times = name_to_times[event_dict["name"]] + if np.all(np.isnan(event_times)): + continue # Skip if all times are NaNs event_type = event_type_name_to_row[event_dict["name"]] for event_time in event_times: events_table.add_row(timestamp=event_time, event_type=event_type) @@ -187,12 +189,16 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, normalize_timestamps: valued_events_table.add_column(name="value", description="Value of the event.") for event_dict in metadata["Behavior"]["ValuedEvents"]: event_times = name_to_times[event_dict["name"]] + if np.all(np.isnan(event_times)): + continue # Skip if all times are NaNs event_values = name_to_values[event_dict["name"]] event_type = event_type_name_to_row[event_dict["name"]] for event_time, event_value in zip(event_times, event_values): valued_events_table.add_row(timestamp=event_time, event_type=event_type, value=event_value) - behavior_module.add(events_table) - behavior_module.add(valued_events_table) + if len(events_table) > 0: + behavior_module.add(events_table) + if len(valued_events_table) > 0: + behavior_module.add(valued_events_table) task = Task(event_types=event_types_table) nwbfile.add_lab_meta_data(task) @@ -207,11 +213,12 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, normalize_timestamps: # Add Epochs Table nwbfile.add_epoch(start_time=trial_start_times[0], stop_time=trial_stop_times[-1], tags=["Active Behavior"]) - nwbfile.add_epoch( - start_time=valued_events_table["timestamp"][0], - stop_time=valued_events_table["timestamp"][-1], - tags=["Passive Listening"], - ) + if len(valued_events_table) > 0: + nwbfile.add_epoch( + start_time=valued_events_table["timestamp"][0], + stop_time=valued_events_table["timestamp"][-1], + tags=["Passive Listening"], + ) # Add Devices for device_kwargs in metadata["Behavior"]["Devices"]: From 43470549183e90c6077c51cc38efb33d12da957d Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 19 Nov 2024 10:52:17 -0800 Subject: [PATCH 14/21] added appropriate types to trials table --- .../zempolich_2024_behaviorinterface.py | 10 +++++++--- .../zempolich_2024/zempolich_2024_metadata.yaml | 14 +++++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py index b6b9235..871fc62 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py @@ -127,15 +127,19 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, normalize_timestamps: trial_start_times = np.array(file["events"]["push"]["time"]).squeeze() trial_stop_times = np.array(file["events"]["push"]["time_end"]).squeeze() trial_is_nan = np.isnan(trial_start_times) | np.isnan(trial_stop_times) - trial_start_times = trial_start_times[np.logical_not(trial_is_nan)] - trial_stop_times = trial_stop_times[np.logical_not(trial_is_nan)] + trial_start_times = trial_start_times[~trial_is_nan] + trial_stop_times = trial_stop_times[~trial_is_nan] if normalize_timestamps: trial_start_times = trial_start_times - starting_timestamp trial_stop_times = trial_stop_times - starting_timestamp for trials_dict in metadata["Behavior"]["Trials"]: name = trials_dict["name"] + dtype = trials_dict["dtype"] trial_array = np.array(file["events"]["push"][name]).squeeze() - name_to_trial_array[name] = trial_array[np.logical_not(trial_is_nan)] + if dtype == "bool": + trial_array[np.isnan(trial_array)] = False + trial_array = np.asarray(trial_array, dtype=dtype) # Can't cast to dtype right away bc bool(nan) = True + name_to_trial_array[name] = trial_array[~trial_is_nan] # Add Data to NWBFile behavior_module = nwb_helpers.get_module( diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml index 61e6239..b2997a9 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml @@ -63,21 +63,29 @@ Behavior: manufacturer: Schneider Lab Trials: - name: rewarded - description: Indicates if trial was rewarded (NaN = trial not rewarded, 1 = trial rewarded). + description: Indicates if trial was rewarded (False = trial not rewarded, True = trial rewarded). + dtype: bool - name: time_reward_s description: Time of reward if rewarded, otherwise NaN. + dtype: float64 - name: opto_trial - description: Indicates if trial was an optogenetic stimulation trial (NaN = non opto trial, 1 = opto trial). + description: Indicates if trial was an optogenetic stimulation trial (False = non opto trial, True = opto trial). + dtype: bool - name: opto_time description: Time of optogenetic stimulation if opto trial, otherwise NaN. + dtype: float64 - name: opto_time_end description: Time of start of optogenetic stimulation if it occurs, otherwise NaN. + dtype: float64 - name: ITI_respect - description: Whether or not trial start obeyed inter trial interval wait time (300ms). + description: Whether or not trial start obeyed inter trial interval wait time of 300ms (False = trial did not obey ITI, True = trial obeyed ITI). + dtype: bool - name: ThresholdVector description: Position of start of the target zone on a given trial (in raw encoder values corresponding to value read out by quadrature encoder). + dtype: float64 - name: endZone_ThresholdVector description: Position of ending/exit position of target zone on a given trial (in raw encoder values corresponding to value read out by quadrature encoder). + dtype: float64 # VideoCamera1: # - name: video_camera_1 # description: Two IR video cameras (AAK CA20 600TVL 2.8MM) are used to monitor the experiments from different angles of interest, allowing for offline analysis of body movements, pupillometry, and other behavioral data if necessary. Camera 1 is a side angle view of the mouse. From b40fcbc5c32ebd726433a37792305ea88b01cd8c Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 19 Nov 2024 12:45:51 -0800 Subject: [PATCH 15/21] added support for m2 --- .../zempolich_2024_convert_session.py | 35 +++++++++++++++++-- .../zempolich_2024_metadata.yaml | 9 ++++- .../zempolich_2024/zempolich_2024_notes.md | 1 + ...ich_2024_open_ephys_recording_interface.py | 9 +++-- .../zempolich_2024_optogeneticinterface.py | 5 +-- 5 files changed, 51 insertions(+), 8 deletions(-) diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py index f9ed1be..ac0b196 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py @@ -3,7 +3,7 @@ from zoneinfo import ZoneInfo import shutil from datetime import datetime -from typing import Optional +from typing import Optional, Literal from neuroconv.utils import load_dict_from_file, dict_deep_update from schneider_lab_to_nwb.zempolich_2024 import Zempolich2024NWBConverter @@ -16,6 +16,7 @@ def session_to_nwb( output_dir_path: str | Path, ephys_folder_path: Optional[str | Path] = None, has_opto: bool = False, + brain_region: Literal["A1", "M2"] = "A1", stub_test: bool = False, ): behavior_file_path = Path(behavior_file_path) @@ -42,7 +43,7 @@ def session_to_nwb( if has_ephys: stream_name = "Signals CH" source_data.update(dict(Recording=dict(folder_path=ephys_folder_path, stream_name=stream_name))) - conversion_options.update(dict(Recording=dict(stub_test=stub_test))) + conversion_options.update(dict(Recording=dict(stub_test=stub_test, brain_region=brain_region))) source_data.update(dict(Sorting=dict(folder_path=ephys_folder_path))) conversion_options.update(dict(Sorting=dict())) @@ -60,7 +61,7 @@ def session_to_nwb( # Add Optogenetic if has_opto: source_data.update(dict(Optogenetic=dict(file_path=behavior_file_path))) - conversion_options.update(dict(Optogenetic=dict())) + conversion_options.update(dict(Optogenetic=dict(brain_region=brain_region))) conversion_options["Behavior"]["normalize_timestamps"] = True # Add Intrinsic Signal Optical Imaging @@ -152,6 +153,34 @@ def main(): stub_test=stub_test, ) + # Example Session M2 Ephys + Behavior + ephys_folder_path = data_dir_path / "M2_EphysFiles" / "m74" / "M2_Day1" + behavior_file_path = data_dir_path / "M2_EphysBehavioralFiles" / "raw_m74_240815_001.mat" + video_folder_path = Path("") + intrinsic_signal_optical_imaging_folder_path = Path("") + session_to_nwb( + ephys_folder_path=ephys_folder_path, + behavior_file_path=behavior_file_path, + video_folder_path=video_folder_path, + intrinsic_signal_optical_imaging_folder_path=intrinsic_signal_optical_imaging_folder_path, + brain_region="M2", + output_dir_path=output_dir_path, + stub_test=stub_test, + ) + + # Example Session M2 Opto + Behavior + behavior_file_path = data_dir_path / "M2_OptoBehavioralFiles" / "raw_m74_240809_001.mat" + video_folder_path = Path("") + intrinsic_signal_optical_imaging_folder_path = Path("") + session_to_nwb( + behavior_file_path=behavior_file_path, + video_folder_path=video_folder_path, + intrinsic_signal_optical_imaging_folder_path=intrinsic_signal_optical_imaging_folder_path, + brain_region="M2", + output_dir_path=output_dir_path, + stub_test=stub_test, + ) + if __name__ == "__main__": main() diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml index b2997a9..1a0e50f 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml @@ -17,6 +17,14 @@ Subject: description: 12-week-old C57BL/6 or VGATChR2-EYFP mice (see Aims approaches below for details) will be used for all behavioral, electrophysiology, and optogenetic experiments. In the VGAT-ChR2-EYFP mouse line, channelrhodopsin (ChR2) is coupled to the vesicular GABA transporter, inducing expression in GABAergic inhibitory neurons ubiquitously across cortex and allowing for real time optogenetic inhibition of brain regions of interest. strain: C57BL/6 +BrainRegion: + A1: + electrode_group_location: Primary Auditory Cortex (A1) + optogenetic_stimulus_site_location: Primary Auditory Cortex (-2.8 AP, 4.2 ML relative to bregma; guided by intrinsic optical imaging) + M2: + electrode_group_location: Secondary Motor Cortex (M2) + optogenetic_stimulus_site_location: Secondary Motor Cortex (1.0-1.5 AP, 0.5-0.7 ML) + Ecephys: Device: - name: MasmanidisSiliconMicroprobe128AxN @@ -25,7 +33,6 @@ Ecephys: ElectrodeGroup: - name: ElectrodeGroup description: ElectrodeGroup for all channels in the recording probe. - location: Primary Auditory Cortex (A1) device: MasmanidisSiliconMicroprobe128AxN ElectricalSeries: - name: ElectricalSeries diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md index b91c3a6..9d3694e 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md @@ -7,6 +7,7 @@ ## Optogenetics - is_opto_trial in the trials table is `np.logical_not(np.isnan(onset_times))` rather than reading from the .mat file to ensure consistency with the onset/offset times. +- injection vs stimulation location(s) for A1 vs M2??? ## Intrinsic Signal Optical Imaging - Just including raw blood vessel image and processed overlay + pixel locations bc including the isoi roi response series would really require an extension for context, but seems like it has limited reuse potential. diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_open_ephys_recording_interface.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_open_ephys_recording_interface.py index b034ab9..b36045f 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_open_ephys_recording_interface.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_open_ephys_recording_interface.py @@ -1,6 +1,7 @@ """Primary class for converting SpikeGadgets Ephys Recordings.""" from pynwb.file import NWBFile import numpy as np +from typing import Literal from neuroconv.datainterfaces import OpenEphysLegacyRecordingInterface from spikeinterface.extractors import OpenEphysLegacyRecordingExtractor @@ -16,12 +17,16 @@ def get_metadata(self) -> dict: metadata["Ecephys"]["Device"] = [] # remove default device return metadata - def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, **conversion_options): + def add_to_nwbfile( + self, nwbfile: NWBFile, metadata: dict, brain_region: Literal["A1", "M2"] = "A1", **conversion_options + ): folder_path = self.source_data["folder_path"] channel_positions = np.load(folder_path / "channel_positions.npy") if True: # TODO: Replace with `if stub_test:` once all channels are present in the data channel_positions = channel_positions[:1, :] - location = metadata["Ecephys"]["ElectrodeGroup"][0]["location"] + location = metadata["BrainRegion"][brain_region]["electrode_group_location"] + for electrode_group in metadata["Ecephys"]["ElectrodeGroup"]: + electrode_group["location"] = location channel_ids = self.recording_extractor.get_channel_ids() self.recording_extractor.set_channel_locations(channel_ids=channel_ids, locations=channel_positions) self.recording_extractor.set_property(key="brain_area", ids=channel_ids, values=[location] * len(channel_ids)) diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_optogeneticinterface.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_optogeneticinterface.py index f194abf..812f24d 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_optogeneticinterface.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_optogeneticinterface.py @@ -1,6 +1,7 @@ """Primary class for converting optogenetic stimulation.""" from pynwb.file import NWBFile from pydantic import FilePath +from typing import Literal import numpy as np from pymatreader import read_mat from pynwb.device import Device @@ -27,7 +28,7 @@ def get_metadata_schema(self) -> dict: metadata_schema = super().get_metadata_schema() return metadata_schema - def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): + def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, brain_region: Literal["A1", "M2"] = "A1"): # Read Data file_path = self.source_data["file_path"] file = read_mat(file_path) @@ -56,7 +57,7 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): # Add OptogeneticStimulusSite site_metadata = metadata["Optogenetics"]["OptogeneticStimulusSite"] - location = f"Injection location: {site_metadata['injection_location']} \n Stimulation location: {site_metadata['stimulation_location']}" + location = metadata["BrainRegion"][brain_region]["optogenetic_stimulus_site_location"] ogen_site = OptogeneticStimulusSite( name=site_metadata["name"], device=device, From 02544e0611150e3e580d3b9e9305cf82d6395ce0 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 19 Nov 2024 12:56:19 -0800 Subject: [PATCH 16/21] updated notes --- .../zempolich_2024/zempolich_2024_notes.md | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md index 9d3694e..4a304a5 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md @@ -16,7 +16,7 @@ - Need device info for 2p microscope and red light laser - Why is the overlaid image flipped left/right compared to the original? -# Temporal Alignment +## Temporal Alignment - For session A1/m53/Day1 (raw_m53_231029_001.mat), - ephys data starts at 1025s with duration 2700s - units table runs from 0-2700 @@ -24,15 +24,20 @@ - events table (toneIN, toneOUT, targetOUT, valve) run from 1191s to 1954s - valued events table () runs from 2017 to 2164 - trials table () runs from 1191 to 1993 - --> conclusion: something is wrong with ephys start time + --> conclusion: something is wrong with ephys start time --> ignoring it - Want to split data into epochs: Active Behavior, Passive Listening, ??? What is happening post-2164? Before 1187s? - For opto sessions, what is the session start time? Ex. What is file['metadata']['session_beginning'] (=129765.7728241)? -- Looks like opto sessions are not temporally aligned (no concurrent ephys and timestamp start at large numbers (142697.1119976)) +- Looks like opto sessions are not temporally aligned (no concurrent ephys and timestamp start at large numbers (142697.1119976)) --> normalizing those sessions times to first encoder timestamp. - -## Data Requests +## Active Requests - Mice sexes -- Remaining data for Grant's project -- More detailed position info for recording probe -- Detailed description of temporal alignment procedure. +- Video for each session and ISOI data for each animal +- pixel locs for ephys +- ISOI device info for 2p microscope and red light laser + +## Questions for Midway Meeting +- injection vs stimulation location(s) for A1 vs M2??? +- Why is the overlaid image flipped left/right compared to the original? +- Want to split data into epochs: Active Behavior, Passive Listening, ??? What is happening post-2164? Before 1187s? +- Double Check: Is it ok to normalize opto sessions to first encoder timestamp? From f869277bb2acd3699973dc2654c3432aae3880dd Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 20 Nov 2024 10:05:29 -0800 Subject: [PATCH 17/21] added verbose --- .../zempolich_2024_convert_session.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py index ac0b196..702e4fd 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py @@ -18,6 +18,7 @@ def session_to_nwb( has_opto: bool = False, brain_region: Literal["A1", "M2"] = "A1", stub_test: bool = False, + verbose: bool = True, ): behavior_file_path = Path(behavior_file_path) video_folder_path = Path(video_folder_path) @@ -42,10 +43,12 @@ def session_to_nwb( # Add Ephys Recording and Sorting if has_ephys: stream_name = "Signals CH" - source_data.update(dict(Recording=dict(folder_path=ephys_folder_path, stream_name=stream_name))) + source_data.update( + dict(Recording=dict(folder_path=ephys_folder_path, stream_name=stream_name, verbose=verbose)) + ) conversion_options.update(dict(Recording=dict(stub_test=stub_test, brain_region=brain_region))) - source_data.update(dict(Sorting=dict(folder_path=ephys_folder_path))) + source_data.update(dict(Sorting=dict(folder_path=ephys_folder_path, verbose=verbose))) conversion_options.update(dict(Sorting=dict())) # Add Behavior @@ -68,7 +71,7 @@ def session_to_nwb( # source_data.update(dict(ISOI=dict(folder_path=intrinsic_signal_optical_imaging_folder_path))) # conversion_options.update(dict(ISOI=dict())) - converter = Zempolich2024NWBConverter(source_data=source_data) + converter = Zempolich2024NWBConverter(source_data=source_data, verbose=verbose) metadata = converter.get_metadata() # Update default metadata with the editable in the corresponding yaml file @@ -122,6 +125,7 @@ def main(): data_dir_path = Path("/Volumes/T7/CatalystNeuro/Schneider/Grant Zempolich Project Data") output_dir_path = Path("/Volumes/T7/CatalystNeuro/Schneider/conversion_nwb") stub_test = False + verbose = False if output_dir_path.exists(): shutil.rmtree(output_dir_path, ignore_errors=True) @@ -138,6 +142,7 @@ def main(): intrinsic_signal_optical_imaging_folder_path=intrinsic_signal_optical_imaging_folder_path, output_dir_path=output_dir_path, stub_test=stub_test, + verbose=verbose, ) # Example Session A1 Ogen + Behavior @@ -151,6 +156,7 @@ def main(): output_dir_path=output_dir_path, has_opto=True, stub_test=stub_test, + verbose=verbose, ) # Example Session M2 Ephys + Behavior @@ -166,6 +172,7 @@ def main(): brain_region="M2", output_dir_path=output_dir_path, stub_test=stub_test, + verbose=verbose, ) # Example Session M2 Opto + Behavior @@ -179,6 +186,7 @@ def main(): brain_region="M2", output_dir_path=output_dir_path, stub_test=stub_test, + verbose=verbose, ) From 87b3ae0ba837f10d48fe38ef60f91b84294d9e9f Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 20 Nov 2024 10:26:06 -0800 Subject: [PATCH 18/21] added convert_all_sessions --- .../schneider_2024_convert_session.py | 168 ------------------ .../zempolich_2024_convert_all_sessions.py | 84 +++++++-- 2 files changed, 70 insertions(+), 182 deletions(-) delete mode 100644 src/schneider_lab_to_nwb/zempolich_2024/schneider_2024_convert_session.py diff --git a/src/schneider_lab_to_nwb/zempolich_2024/schneider_2024_convert_session.py b/src/schneider_lab_to_nwb/zempolich_2024/schneider_2024_convert_session.py deleted file mode 100644 index f44d076..0000000 --- a/src/schneider_lab_to_nwb/zempolich_2024/schneider_2024_convert_session.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Primary script to run to convert an entire session for of data using the NWBConverter.""" -from pathlib import Path -from zoneinfo import ZoneInfo -import shutil -import numpy as np -from datetime import datetime -from typing import Optional - -from neuroconv.utils import load_dict_from_file, dict_deep_update -from schneider_lab_to_nwb.zempolich_2024 import Zempolich2024NWBConverter - - -def session_to_nwb( - behavior_file_path: str | Path, - video_folder_path: str | Path, - intrinsic_signal_optical_imaging_folder_path: str | Path, - output_dir_path: str | Path, - ephys_folder_path: Optional[str | Path] = None, - has_opto: bool = False, - stub_test: bool = False, -): - behavior_file_path = Path(behavior_file_path) - video_folder_path = Path(video_folder_path) - intrinsic_signal_optical_imaging_folder_path = Path(intrinsic_signal_optical_imaging_folder_path) - output_dir_path = Path(output_dir_path) - video_file_paths = [ - file_path for file_path in video_folder_path.glob("*.mp4") if not file_path.name.startswith("._") - ] - video_file_paths = sorted(video_file_paths) - if stub_test: - output_dir_path = output_dir_path / "nwb_stub" - output_dir_path.mkdir(parents=True, exist_ok=True) - if ephys_folder_path is None: - has_ephys = False - else: - has_ephys = True - ephys_folder_path = Path(ephys_folder_path) - - source_data = dict() - conversion_options = dict() - - # Add Ephys Recording and Sorting - if has_ephys: - stream_name = "Signals CH" - source_data.update(dict(Recording=dict(folder_path=ephys_folder_path, stream_name=stream_name))) - conversion_options.update(dict(Recording=dict(stub_test=stub_test))) - - source_data.update(dict(Sorting=dict(folder_path=ephys_folder_path))) - conversion_options.update(dict(Sorting=dict())) - - # Add Behavior - source_data.update(dict(Behavior=dict(file_path=behavior_file_path))) - conversion_options.update(dict(Behavior=dict())) - - # Add Video(s) - # for i, video_file_path in enumerate(video_file_paths): - # metadata_key_name = f"VideoCamera{i+1}" - # source_data.update({metadata_key_name: dict(file_paths=[video_file_path], metadata_key_name=metadata_key_name)}) - # conversion_options.update({metadata_key_name: dict()}) - - # Add Optogenetic - if has_opto: - source_data.update(dict(Optogenetic=dict(file_path=behavior_file_path))) - conversion_options.update(dict(Optogenetic=dict())) - - # Add Intrinsic Signal Optical Imaging - # source_data.update(dict(ISOI=dict(folder_path=intrinsic_signal_optical_imaging_folder_path))) - # conversion_options.update(dict(ISOI=dict())) - - converter = Zempolich2024NWBConverter(source_data=source_data) - metadata = converter.get_metadata() - - # Update default metadata with the editable in the corresponding yaml file - editable_metadata_path = Path(__file__).parent / "schneider_2024_metadata.yaml" - editable_metadata = load_dict_from_file(editable_metadata_path) - metadata = dict_deep_update(metadata, editable_metadata) - if has_ephys: - folder_name = ( - source_data["Recording"]["folder_path"].parent.name + "/" + source_data["Recording"]["folder_path"].name - ) - folder_name_to_start_datetime = metadata["Ecephys"].pop("folder_name_to_start_datetime") - if folder_name in folder_name_to_start_datetime.keys(): - metadata["NWBFile"]["session_start_time"] = datetime.fromisoformat( - folder_name_to_start_datetime[folder_name] - ) - else: - metadata["NWBFile"]["session_start_time"] = datetime.strptime(behavior_file_path.name.split("_")[2], "%y%m%d") - - # Add datetime to conversion - EST = ZoneInfo("US/Eastern") - metadata["NWBFile"]["session_start_time"] = metadata["NWBFile"]["session_start_time"].replace(tzinfo=EST) - - metadata["Subject"]["subject_id"] = "a_subject_id" # Modify here or in the yaml file - if has_ephys: - conversion_options["Sorting"]["units_description"] = metadata["Sorting"]["units_description"] - - # Add electrode metadata - if has_ephys: - channel_positions = np.load(ephys_folder_path / "channel_positions.npy") - if True: # stub_test: SWITCH BACK TO stub_test WHEN ALL CHANNELS ARE PRESENT - channel_positions = channel_positions[:1, :] - location = metadata["Ecephys"]["ElectrodeGroup"][0]["location"] - channel_ids = converter.data_interface_objects["Recording"].recording_extractor.get_channel_ids() - converter.data_interface_objects["Recording"].recording_extractor.set_channel_locations( - channel_ids=channel_ids, locations=channel_positions - ) - converter.data_interface_objects["Recording"].recording_extractor.set_property( - key="brain_area", - ids=channel_ids, - values=[location] * len(channel_ids), - ) - converter.data_interface_objects["Recording"].recording_extractor._recording_segments[0].t_start = 0.0 - metadata["Ecephys"]["Device"] = editable_metadata["Ecephys"]["Device"] - - # # Overwrite video metadata - # for i, video_file_path in enumerate(video_file_paths): - # metadata_key_name = f"VideoCamera{i+1}" - # metadata["Behavior"][metadata_key_name] = editable_metadata["Behavior"][metadata_key_name] - - subject_id = behavior_file_path.name.split("_")[1] - session_id = behavior_file_path.name.split("_")[2] - nwbfile_path = output_dir_path / f"sub-{subject_id}_ses-{session_id}.nwb" - metadata["NWBFile"]["session_id"] = session_id - metadata["Subject"]["subject_id"] = subject_id - - # Run conversion - converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options) - - -def main(): - # Parameters for conversion - data_dir_path = Path("/Volumes/T7/CatalystNeuro/Schneider/Grant Zempolich Project Data") - output_dir_path = Path("/Volumes/T7/CatalystNeuro/Schneider/conversion_nwb") - stub_test = False - - if output_dir_path.exists(): - shutil.rmtree(output_dir_path, ignore_errors=True) - - # Example Session A1 Ephys + Behavior - ephys_folder_path = data_dir_path / "A1_EphysFiles" / "m53" / "Day1_A1" - behavior_file_path = data_dir_path / "A1_EphysBehavioralFiles" / "raw_m53_231029_001.mat" - video_folder_path = Path("") - intrinsic_signal_optical_imaging_folder_path = Path("") - session_to_nwb( - ephys_folder_path=ephys_folder_path, - behavior_file_path=behavior_file_path, - video_folder_path=video_folder_path, - intrinsic_signal_optical_imaging_folder_path=intrinsic_signal_optical_imaging_folder_path, - output_dir_path=output_dir_path, - stub_test=stub_test, - ) - - # Example Session A1 Ogen + Behavior - behavior_file_path = data_dir_path / "A1_OptoBehavioralFiles" / "raw_m53_231013_001.mat" - video_folder_path = Path("") - intrinsic_signal_optical_imaging_folder_path = Path("") - session_to_nwb( - behavior_file_path=behavior_file_path, - video_folder_path=video_folder_path, - intrinsic_signal_optical_imaging_folder_path=intrinsic_signal_optical_imaging_folder_path, - output_dir_path=output_dir_path, - has_opto=True, - stub_test=stub_test, - ) - - -if __name__ == "__main__": - main() diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_all_sessions.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_all_sessions.py index f256fa9..467ca54 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_all_sessions.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_all_sessions.py @@ -4,8 +4,9 @@ from pprint import pformat import traceback from tqdm import tqdm +import shutil -from .zempolich_2024_convert_session import session_to_nwb +from schneider_lab_to_nwb.zempolich_2024.zempolich_2024_convert_session import session_to_nwb def dataset_to_nwb( @@ -29,16 +30,16 @@ def dataset_to_nwb( Whether to print verbose output, by default True """ data_dir_path = Path(data_dir_path) - session_to_nwb_kwargs_per_session = get_session_to_nwb_kwargs_per_session( - data_dir_path=data_dir_path, - ) + output_dir_path = Path(output_dir_path) + session_to_nwb_kwargs_per_session = get_session_to_nwb_kwargs_per_session(data_dir_path=data_dir_path) futures = [] with ProcessPoolExecutor(max_workers=max_workers) as executor: for session_to_nwb_kwargs in session_to_nwb_kwargs_per_session: session_to_nwb_kwargs["output_dir_path"] = output_dir_path session_to_nwb_kwargs["verbose"] = verbose - exception_file_path = data_dir_path / f"ERROR_.txt" # Add error file path here + nwbfile_name = get_nwbfile_name_from_kwargs(session_to_nwb_kwargs) + exception_file_path = output_dir_path / f"ERROR_{nwbfile_name}.txt" futures.append( executor.submit( safe_session_to_nwb, @@ -50,6 +51,14 @@ def dataset_to_nwb( pass +def get_nwbfile_name_from_kwargs(session_to_nwb_kwargs): + behavior_file_path = session_to_nwb_kwargs["behavior_file_path"] + subject_id = behavior_file_path.name.split("_")[1] + session_id = behavior_file_path.name.split("_")[2] + nwbfile_name = f"sub-{subject_id}_ses-{session_id}.nwb" + return nwbfile_name + + def safe_session_to_nwb(*, session_to_nwb_kwargs: dict, exception_file_path: str | Path): """Convert a session to NWB while handling any errors by recording error messages to the exception_file_path. @@ -85,20 +94,67 @@ def get_session_to_nwb_kwargs_per_session( list[dict[str, Any]] A list of dictionaries containing the kwargs for session_to_nwb for each session. """ - ##### - # # Implement this function to return the kwargs for session_to_nwb for each session - # This can be a specific list with hard-coded sessions, a path expansion or any conversion specific logic that you might need - ##### - raise NotImplementedError + a1_ephys_path = data_dir_path / "A1_EphysFiles" + a1_ephys_behavior_path = data_dir_path / "A1_EphysBehavioralFiles" + a1_opto_path = data_dir_path / "A1_OptoBehavioralFiles" + m2_ephys_path = data_dir_path / "M2_EphysFiles" + m2_ephys_behavior_path = data_dir_path / "M2_EphysBehavioralFiles" + m2_opto_path = data_dir_path / "M2_OptoBehavioralFiles" + + a1_kwargs = get_brain_region_kwargs( + ephys_path=a1_ephys_path, + ephys_behavior_path=a1_ephys_behavior_path, + opto_path=a1_opto_path, + brain_region="A1", + ) + m2_kwargs = get_brain_region_kwargs( + ephys_path=m2_ephys_path, + ephys_behavior_path=m2_ephys_behavior_path, + opto_path=m2_opto_path, + brain_region="M2", + ) + session_to_nwb_kwargs_per_session = a1_kwargs + m2_kwargs + + return session_to_nwb_kwargs_per_session + + +def get_brain_region_kwargs(ephys_path, ephys_behavior_path, opto_path, brain_region): + session_to_nwb_kwargs_per_session = [] + for subject_dir in ephys_path.iterdir(): + subject_id = subject_dir.name + matched_behavior_paths = sorted(ephys_behavior_path.glob(f"raw_{subject_id}_*.mat")) + sorted_session_dirs = sorted(subject_dir.iterdir()) + for ephys_folder_path, behavior_file_path in zip(sorted_session_dirs, matched_behavior_paths): + session_to_nwb_kwargs = dict( + ephys_folder_path=ephys_folder_path, + behavior_file_path=behavior_file_path, + brain_region=brain_region, + intrinsic_signal_optical_imaging_folder_path="", # TODO: Add intrinsic signal optical imaging folder path + video_folder_path="", # TODO: Add video folder path + ) + session_to_nwb_kwargs_per_session.append(session_to_nwb_kwargs) + for behavior_file_path in opto_path.iterdir(): + session_to_nwb_kwargs = dict( + behavior_file_path=behavior_file_path, + brain_region=brain_region, + has_opto=True, + intrinsic_signal_optical_imaging_folder_path="", # TODO: Add intrinsic signal optical imaging folder path + video_folder_path="", # TODO: Add video folder path + ) + session_to_nwb_kwargs_per_session.append(session_to_nwb_kwargs) + return session_to_nwb_kwargs_per_session if __name__ == "__main__": # Parameters for conversion - data_dir_path = Path("/Directory/With/Raw/Formats/") - output_dir_path = Path("~/conversion_nwb/") - max_workers = 1 - verbose = False + data_dir_path = Path("/Volumes/T7/CatalystNeuro/Schneider/Grant Zempolich Project Data") + output_dir_path = Path("/Volumes/T7/CatalystNeuro/Schneider/conversion_nwb") + max_workers = 4 + if output_dir_path.exists(): + shutil.rmtree( + output_dir_path, ignore_errors=True + ) # ignore errors due to MacOS race condition (https://github.com/python/cpython/issues/81441) dataset_to_nwb( data_dir_path=data_dir_path, From 5e2d74b76b1a111322dad0e10c95f54c0a04680a Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 20 Nov 2024 11:55:14 -0800 Subject: [PATCH 19/21] added start times for ambiguous sessions --- .../zempolich_2024/zempolich_2024_metadata.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml index 1a0e50f..0e7ef7b 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml @@ -39,6 +39,14 @@ Ecephys: description: Recording of AC neural responses in mice performing this behavioral task will utilize dense 128-channel recording probes (Masmanidis Lab). These recording probes span a depth ~1mm, allowing for sampling of all layers of cortex. Electrophysiology data will be recorded using OpenEphys Acquisition Board v2.4 and associated OpenEphys GUI software. folder_name_to_start_datetime: m53/Day1_A1: 2023-10-29T16:56:01 + m54/Day1_A1: 2023-10-29T18:18:03 + m65/Day2_A1: 2023-10-26T18:49:04 + m66/Day1_A1: 2023-10-25T18:41:03 # could be 2023-10-25T18:04:13 instead + m67/Day2_A1: 2023-10-28T15:54:01 # could be 2023-10-28T15:05:14 instead + m71/Day2_A1: 2023-11-01T18:04:30 + m72/Day2_A1: 2023-10-21T20:03:27 # could be 2023-10-21T20:32:07 instead + m79/M2_Day1: 2024-08-19T15:58:07 + m81/M2_Day1: 2024-08-17T17:59:09 Behavior: Module: From 73914e20b90d9aa3f8a88a1492509ad8a80384cb Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 20 Nov 2024 11:56:36 -0800 Subject: [PATCH 20/21] added start times for ambiguous sessions --- .../zempolich_2024/zempolich_2024_metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml index 0e7ef7b..433fada 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml @@ -43,7 +43,7 @@ Ecephys: m65/Day2_A1: 2023-10-26T18:49:04 m66/Day1_A1: 2023-10-25T18:41:03 # could be 2023-10-25T18:04:13 instead m67/Day2_A1: 2023-10-28T15:54:01 # could be 2023-10-28T15:05:14 instead - m71/Day2_A1: 2023-11-01T18:04:30 + m71/Day2_A1: 2023-11-01T18:04:30 # could be 2023-11-01T18:43:00 instead m72/Day2_A1: 2023-10-21T20:03:27 # could be 2023-10-21T20:32:07 instead m79/M2_Day1: 2024-08-19T15:58:07 m81/M2_Day1: 2024-08-17T17:59:09 From 752a6c6bec0d2c3f852d614b0531888febf90b74 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 25 Nov 2024 09:00:57 -0800 Subject: [PATCH 21/21] added print for skipped events --- .../zempolich_2024_behaviorinterface.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py index 871fc62..0c6a65f 100644 --- a/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py +++ b/src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_behaviorinterface.py @@ -89,7 +89,9 @@ def get_metadata_schema(self) -> dict: } return metadata_schema - def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, normalize_timestamps: bool = False): + def add_to_nwbfile( + self, nwbfile: NWBFile, metadata: dict, normalize_timestamps: bool = False, verbose: bool = False + ): # Read Data file_path = self.source_data["file_path"] file = read_mat(file_path) @@ -181,6 +183,10 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, normalize_timestamps: for event_dict in metadata["Behavior"]["Events"]: event_times = name_to_times[event_dict["name"]] if np.all(np.isnan(event_times)): + if verbose: + print( + f"An event provided in the metadata ({event_dict['name']}) will be skipped because no times were found." + ) continue # Skip if all times are NaNs event_type = event_type_name_to_row[event_dict["name"]] for event_time in event_times: @@ -194,6 +200,10 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict, normalize_timestamps: for event_dict in metadata["Behavior"]["ValuedEvents"]: event_times = name_to_times[event_dict["name"]] if np.all(np.isnan(event_times)): + if verbose: + print( + f"An event provided in the metadata ({event_dict['name']}) will be skipped because no times were found." + ) continue # Skip if all times are NaNs event_values = name_to_values[event_dict["name"]] event_type = event_type_name_to_row[event_dict["name"]]