From 3ab8de39293b4f35102d4d2f0a2d9fa350818807 Mon Sep 17 00:00:00 2001 From: Georg Raiser Date: Wed, 11 Dec 2024 14:02:48 +0000 Subject: [PATCH] revision argument in all datainterfaces --- ...inwide_map_processed_only_local_testing.py | 16 ++-- ...rt_brainwide_map_raw_only_local_testing.py | 77 ++++++++++--------- .../datainterfaces/_brainwide_map_trials.py | 5 +- .../datainterfaces/_ibl_sorting_extractor.py | 7 +- .../_ibl_streaming_interface.py | 3 +- src/ibl_to_nwb/datainterfaces/_lick_times.py | 5 +- .../datainterfaces/_pose_estimation.py | 21 ++--- .../datainterfaces/_pupil_tracking.py | 5 +- .../datainterfaces/_roi_motion_energy.py | 5 +- .../datainterfaces/_wheel_movement.py | 7 +- 10 files changed, 83 insertions(+), 68 deletions(-) diff --git a/src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only_local_testing.py b/src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only_local_testing.py index 3961870..bddd4bd 100644 --- a/src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only_local_testing.py +++ b/src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only_local_testing.py @@ -34,6 +34,7 @@ cleanup: bool = False # assert len(os.environ.get("DANDI_API_KEY", "")) > 0, "Run `export DANDI_API_KEY=...`!" +revision = None nwbfile_path.parent.mkdir(exist_ok=True) @@ -52,32 +53,29 @@ # These interfaces should always be present in source data data_interfaces.append(IblSortingInterface(session=session, cache_folder=cache_folder / "sorting")) -data_interfaces.append(BrainwideMapTrialsInterface(one=session_one, session=session)) -data_interfaces.append(WheelInterface(one=session_one, session=session)) +data_interfaces.append(BrainwideMapTrialsInterface(one=session_one, session=session, revision=revision)) +data_interfaces.append(WheelInterface(one=session_one, session=session, revision=revision)) # These interfaces may not be present; check if they are before adding to list pose_estimation_files = session_one.list_datasets(eid=session, filename="*.dlc*") for pose_estimation_file in pose_estimation_files: camera_name = pose_estimation_file.replace("alf/_ibl_", "").replace(".dlc.pqt", "") data_interfaces.append( - # IblPoseEstimationInterface( - # one=session_one, session=session, camera_name=camera_name, include_pose=True, include_video=False - # ) - IblPoseEstimationInterface(one=session_one, session=session, camera_name=camera_name) + IblPoseEstimationInterface(one=session_one, session=session, camera_name=camera_name, revision=revision) ) pupil_tracking_files = session_one.list_datasets(eid=session, filename="*features*") for pupil_tracking_file in pupil_tracking_files: camera_name = pupil_tracking_file.replace("alf/_ibl_", "").replace(".features.pqt", "") - data_interfaces.append(PupilTrackingInterface(one=session_one, session=session, camera_name=camera_name)) + data_interfaces.append(PupilTrackingInterface(one=session_one, session=session, camera_name=camera_name, revision=revision)) roi_motion_energy_files = session_one.list_datasets(eid=session, filename="*ROIMotionEnergy.npy*") for roi_motion_energy_file in roi_motion_energy_files: camera_name = roi_motion_energy_file.replace("alf/", "").replace(".ROIMotionEnergy.npy", "") - data_interfaces.append(RoiMotionEnergyInterface(one=session_one, session=session, camera_name=camera_name)) + data_interfaces.append(RoiMotionEnergyInterface(one=session_one, session=session, camera_name=camera_name, revision=revision)) if session_one.list_datasets(eid=session, collection="alf", filename="licks*"): - data_interfaces.append(LickInterface(one=session_one, session=session)) + data_interfaces.append(LickInterface(one=session_one, session=session, revision=revision)) # Run conversion session_converter = BrainwideMapConverter( diff --git a/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only_local_testing.py b/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only_local_testing.py index 2695525..3c162c8 100644 --- a/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only_local_testing.py +++ b/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only_local_testing.py @@ -1,20 +1,22 @@ +# %% from pathlib import Path - from one.api import ONE - from ibl_to_nwb.converters import BrainwideMapConverter, IblSpikeGlxConverter from ibl_to_nwb.datainterfaces import RawVideoInterface -# session_id = "d32876dd-8303-4720-8e7e-20678dc2fd71" -session_id = "caa5dddc-9290-4e27-9f5e-575ba3598614" # a BWM session with dual probe -data_folder = Path( - "/media/georg/openlab/Downloads/ONE/openalyx.internationalbrainlab.org/steinmetzlab/Subjects/NR_0031/2023-07-14/001" -) -spikeglx_source_folder_path = data_folder / "raw_ephys_data" +# eid = "d32876dd-8303-4720-8e7e-20678dc2fd71" +eid = "caa5dddc-9290-4e27-9f5e-575ba3598614" # a BWM session with dual probe + +# %% +# one_cache_folder = '/home/georg/ibl_scratch/ibl_conversion/caa5dddc-9290-4e27-9f5e-575ba3598614/cache' +# data_folder = Path( +# "/media/georg/openlab/Downloads/ONE/openalyx.internationalbrainlab.org/steinmetzlab/Subjects/NR_0031/2023-07-14/001" +# ) +# spikeglx_source_folder_path = data_folder / "raw_ephys_data" # Specify the revision of the pose estimation data # Setting to 'None' will use whatever the latest released revision is -revision = None +# revision = None # base_path = Path("E:/IBL") base_path = Path.home() / "ibl_scratch" # local directory @@ -23,50 +25,55 @@ nwbfiles_folder_path.mkdir(exist_ok=True) # Initialize IBL (ONE) client to download processed data for this session -one_cache_folder_path = base_path / "cache" -ibl_client = ONE( +# one_cache_folder_path = base_path / "cache" +one_cache_folder_path = "/home/georg/ibl_scratch/ibl_conversion/caa5dddc-9290-4e27-9f5e-575ba3598614/cache" +one = ONE( base_url="https://openalyx.internationalbrainlab.org", password="international", silent=True, cache_dir=one_cache_folder_path, ) -# Specify the path to the SpikeGLX files on the server but use ONE API for timestamps data_interfaces = [] -# spikeglx_source_folder_path = Path("D:/example_data/ephy_testing_data/spikeglx/Noise4Sam_g0") -spikeglx_subconverter = IblSpikeGlxConverter(folder_path=spikeglx_source_folder_path, one=ibl_client, eid=session_id) -data_interfaces.append(spikeglx_subconverter) +# %% ephys +# session_folder = one.eid2path(eid) +# spikeglx_source_folder_path = session_folder / 'raw_ephys_data' -# # Raw video takes some special handling -# metadata_retrieval = BrainwideMapConverter(one=ibl_client, session=session_id, data_interfaces=[], verbose=False) -# subject_id = metadata_retrieval.get_metadata()["Subject"]["subject_id"] -# pose_estimation_files = ibl_client.list_datasets(eid=session_id, filename="*.dlc*") -# for pose_estimation_file in pose_estimation_files: -# camera_name = pose_estimation_file.replace("alf/_ibl_", "").replace(".dlc.pqt", "") +# Specify the path to the SpikeGLX files on the server but use ONE API for timestamps +# spikeglx_subconverter = IblSpikeGlxConverter(folder_path=spikeglx_source_folder_path, one=one, eid=eid) +# data_interfaces.append(spikeglx_subconverter) + + +# %% video +# Raw video takes some special handling +metadata_retrieval = BrainwideMapConverter(one=one, session=eid, data_interfaces=[], verbose=False) +subject_id = metadata_retrieval.get_metadata()["Subject"]["subject_id"] -# video_interface = RawVideoInterface( -# nwbfiles_folder_path=nwbfiles_folder_path, -# subject_id=subject_id, -# one=ibl_client, -# session=session_id, -# camera_name=camera_name, -# ) -# data_interfaces.append(video_interface) +pose_estimation_files = one.list_datasets(eid=eid, filename="*.dlc*") +for pose_estimation_file in pose_estimation_files: + camera_name = pose_estimation_file.replace("alf/_ibl_", "").replace(".dlc.pqt", "") + + video_interface = RawVideoInterface( + nwbfiles_folder_path=nwbfiles_folder_path, + subject_id=subject_id, + one=one, + session=eid, + camera_name=camera_name, + ) + data_interfaces.append(video_interface) # Run conversion -session_converter = BrainwideMapConverter( - one=ibl_client, session=session_id, data_interfaces=data_interfaces, verbose=False -) +session_converter = BrainwideMapConverter(one=one, session=eid, data_interfaces=data_interfaces, verbose=False) metadata = session_converter.get_metadata() -metadata["NWBFile"]["session_id"] = metadata["NWBFile"]["session_id"] +metadata["NWBFile"]["eid"] = metadata["NWBFile"]["eid"] subject_id = metadata["Subject"]["subject_id"] subject_folder_path = nwbfiles_folder_path / f"sub-{subject_id}" subject_folder_path.mkdir(exist_ok=True) -nwbfile_path = subject_folder_path / f"sub-{subject_id}_ses-{session_id}_desc-raw_ecephys+image.nwb" +nwbfile_path = subject_folder_path / f"sub-{subject_id}_ses-{eid}_desc-video.nwb" session_converter.run_conversion( nwbfile_path=nwbfile_path, @@ -75,4 +82,4 @@ ) # TODO: add some kind of raw-specific check -# check_written_nwbfile_for_consistency(one=ibl_client, nwbfile_path=nwbfile_path) +# check_written_nwbfile_for_consistency(one=one, nwbfile_path=nwbfile_path) diff --git a/src/ibl_to_nwb/datainterfaces/_brainwide_map_trials.py b/src/ibl_to_nwb/datainterfaces/_brainwide_map_trials.py index 723b7d5..c7594f7 100644 --- a/src/ibl_to_nwb/datainterfaces/_brainwide_map_trials.py +++ b/src/ibl_to_nwb/datainterfaces/_brainwide_map_trials.py @@ -9,9 +9,10 @@ class BrainwideMapTrialsInterface(BaseDataInterface): - def __init__(self, one: ONE, session: str): + def __init__(self, one: ONE, session: str, revision: str | None = None): self.one = one self.session = session + self.revision = one.list_revisions(session)[-1] if revision is None else revision def get_metadata(self) -> dict: metadata = super().get_metadata() @@ -20,7 +21,7 @@ def get_metadata(self) -> dict: return metadata def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): - trials = self.one.load_object(id=self.session, obj="trials", collection="alf") + trials = self.one.load_object(id=self.session, obj="trials", collection="alf", revision=self.revision) column_ordering = [ "choice", diff --git a/src/ibl_to_nwb/datainterfaces/_ibl_sorting_extractor.py b/src/ibl_to_nwb/datainterfaces/_ibl_sorting_extractor.py index 38cbc7e..b698da3 100644 --- a/src/ibl_to_nwb/datainterfaces/_ibl_sorting_extractor.py +++ b/src/ibl_to_nwb/datainterfaces/_ibl_sorting_extractor.py @@ -16,7 +16,7 @@ class IblSortingExtractor(BaseSorting): installation_mesg = "" name = "iblsorting" - def __init__(self, session: str, cache_folder: Optional[DirectoryPath] = None): + def __init__(self, session: str, cache_folder: Optional[DirectoryPath] = None, revision=None): from brainbox.io.one import SpikeSortingLoader from iblatlas.atlas import AllenAtlas from iblatlas.regions import BrainRegions @@ -28,6 +28,9 @@ def __init__(self, session: str, cache_folder: Optional[DirectoryPath] = None): silent=True, cache_dir=cache_folder, ) + if revision is None: # latest + revision = one.list_revisions(session)[-1] + atlas = AllenAtlas() brain_regions = BrainRegions() @@ -45,7 +48,7 @@ def __init__(self, session: str, cache_folder: Optional[DirectoryPath] = None): for probe_name in probe_names: sorting_loader = SpikeSortingLoader(eid=session, one=one, pname=probe_name, atlas=atlas) sorting_loaders.update({probe_name: sorting_loader}) - spikes, clusters, channels = sorting_loader.load_spike_sorting() + spikes, clusters, channels = sorting_loader.load_spike_sorting(revision=revision) # cluster_ids.extend(list(np.array(clusters["metrics"]["cluster_id"]) + unit_id_per_probe_shift)) number_of_units = len(np.unique(spikes["clusters"])) cluster_ids.extend(list(np.arange(number_of_units).astype("int32") + unit_id_per_probe_shift)) diff --git a/src/ibl_to_nwb/datainterfaces/_ibl_streaming_interface.py b/src/ibl_to_nwb/datainterfaces/_ibl_streaming_interface.py index f8aac0b..5064633 100644 --- a/src/ibl_to_nwb/datainterfaces/_ibl_streaming_interface.py +++ b/src/ibl_to_nwb/datainterfaces/_ibl_streaming_interface.py @@ -81,7 +81,8 @@ def __init__(self, **kwargs): self.recording_extractor.set_property(key="ibl_y", values=ibl_coords[:, 1]) self.recording_extractor.set_property(key="ibl_z", values=ibl_coords[:, 2]) self.recording_extractor.set_property( # SpikeInterface refers to this as 'brain_area' - key="brain_area", values=list(channels["acronym"]) # NeuroConv remaps to 'location', a required field + key="brain_area", + values=list(channels["acronym"]), # NeuroConv remaps to 'location', a required field ) # Acronyms are symmetric, do not differentiate hemisphere self.recording_extractor.set_property( key="beryl_location", diff --git a/src/ibl_to_nwb/datainterfaces/_lick_times.py b/src/ibl_to_nwb/datainterfaces/_lick_times.py index 76f9a9e..375f854 100644 --- a/src/ibl_to_nwb/datainterfaces/_lick_times.py +++ b/src/ibl_to_nwb/datainterfaces/_lick_times.py @@ -7,12 +7,13 @@ class LickInterface(BaseDataInterface): - def __init__(self, one: ONE, session: str): + def __init__(self, one: ONE, session: str, revision: str | None = None): self.one = one self.session = session + self.revision = one.list_revisions(session)[-1] if revision is None else revision def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): - licks = self.one.load_object(id=self.session, obj="licks", collection="alf") + licks = self.one.load_object(id=self.session, obj="licks", collection="alf", revision=self.revision) lick_events_table = DynamicTable( name="LickTimes", diff --git a/src/ibl_to_nwb/datainterfaces/_pose_estimation.py b/src/ibl_to_nwb/datainterfaces/_pose_estimation.py index abf30d3..94946b7 100644 --- a/src/ibl_to_nwb/datainterfaces/_pose_estimation.py +++ b/src/ibl_to_nwb/datainterfaces/_pose_estimation.py @@ -37,17 +37,18 @@ def __init__( self.revision = revision if self.revision is None: - session_files = self.one.list_datasets(eid=self.session, filename=f"*{self.camera_name}.dlc*") - revision_datetime_format = "%Y-%m-%d" - revisions = [ - datetime.strptime(session_file.split("#")[1], revision_datetime_format) - for session_file in session_files - if "#" in session_file - ] + self.revision = one.list_revisions(session)[-1] + # session_files = self.one.list_datasets(eid=self.session, filename=f"*{self.camera_name}.dlc*") + # revision_datetime_format = "%Y-%m-%d" + # revisions = [ + # datetime.strptime(session_file.split("#")[1], revision_datetime_format) + # for session_file in session_files + # if "#" in session_file + # ] - if any(revisions): - most_recent = max(revisions) - self.revision = most_recent.strftime("%Y-%m-%d") + # if any(revisions): + # most_recent = max(revisions) + # self.revision = most_recent.strftime("%Y-%m-%d") def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict) -> None: camera_data = self.one.load_object( diff --git a/src/ibl_to_nwb/datainterfaces/_pupil_tracking.py b/src/ibl_to_nwb/datainterfaces/_pupil_tracking.py index c307ef6..0477795 100644 --- a/src/ibl_to_nwb/datainterfaces/_pupil_tracking.py +++ b/src/ibl_to_nwb/datainterfaces/_pupil_tracking.py @@ -12,10 +12,11 @@ class PupilTrackingInterface(BaseDataInterface): - def __init__(self, one: ONE, session: str, camera_name: str): + def __init__(self, one: ONE, session: str, camera_name: str, revision: str | None = None): self.one = one self.session = session self.camera_name = camera_name + self.revision = one.list_revisions(session)[-1] if revision is None else revision def get_metadata(self) -> dict: metadata = super().get_metadata() @@ -28,7 +29,7 @@ def get_metadata(self) -> dict: def add_to_nwbfile(self, nwbfile, metadata: dict): left_or_right = self.camera_name[:5].rstrip("C") - camera_data = self.one.load_object(id=self.session, obj=self.camera_name, collection="alf") + camera_data = self.one.load_object(id=self.session, obj=self.camera_name, collection="alf", revision=self.revision) pupil_time_series = list() for ibl_key in ["pupilDiameter_raw", "pupilDiameter_smooth"]: diff --git a/src/ibl_to_nwb/datainterfaces/_roi_motion_energy.py b/src/ibl_to_nwb/datainterfaces/_roi_motion_energy.py index 4218647..8ea21d3 100644 --- a/src/ibl_to_nwb/datainterfaces/_roi_motion_energy.py +++ b/src/ibl_to_nwb/datainterfaces/_roi_motion_energy.py @@ -7,17 +7,18 @@ class RoiMotionEnergyInterface(BaseDataInterface): - def __init__(self, one: ONE, session: str, camera_name: str): + def __init__(self, one: ONE, session: str, camera_name: str, revision: str | None = None): self.one = one self.session = session self.camera_name = camera_name + self.revision = one.list_revisions(session)[-1] if revision is None else revision def add_to_nwbfile(self, nwbfile, metadata: dict): left_right_or_body = self.camera_name[:5].rstrip("C") camera_data = self.one.load_object(id=self.session, obj=self.camera_name, collection="alf") motion_energy_video_region = self.one.load_object( - id=self.session, obj=f"{left_right_or_body}ROIMotionEnergy", collection="alf" + id=self.session, obj=f"{left_right_or_body}ROIMotionEnergy", collection="alf", revision=self.revision ) width, height, x, y = motion_energy_video_region["position"] diff --git a/src/ibl_to_nwb/datainterfaces/_wheel_movement.py b/src/ibl_to_nwb/datainterfaces/_wheel_movement.py index 234b307..00ec234 100644 --- a/src/ibl_to_nwb/datainterfaces/_wheel_movement.py +++ b/src/ibl_to_nwb/datainterfaces/_wheel_movement.py @@ -11,9 +11,10 @@ class WheelInterface(BaseDataInterface): - def __init__(self, one: ONE, session: str): + def __init__(self, one: ONE, session: str, revision: str | None = None): self.one = one self.session = session + self.revision = one.list_revisions(session) if revision is None else revision def get_metadata(self) -> dict: metadata = super().get_metadata() @@ -23,8 +24,8 @@ def get_metadata(self) -> dict: return metadata def add_to_nwbfile(self, nwbfile, metadata: dict): - wheel_moves = self.one.load_object(id=self.session, obj="wheelMoves", collection="alf") - wheel = self.one.load_object(id=self.session, obj="wheel", collection="alf") + wheel_moves = self.one.load_object(id=self.session, obj="wheelMoves", collection="alf", revision=self.revision) + wheel = self.one.load_object(id=self.session, obj="wheel", collection="alf", revision=self.revision) # Estimate velocity and acceleration interpolation_frequency = 1000.0 # Hz