Conversion test

.gitignore

@@ -134,3 +134,4 @@ dmypy.json
 #misc
 endpoint_schemas/
 tests/
+src/local

src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only.py

@@ -14,13 +14,15 @@
 )
 from ibl_to_nwb.testing import check_written_nwbfile_for_consistency
 
-session_id = "d32876dd-8303-4720-8e7e-20678dc2fd71"
+# session_id = "d32876dd-8303-4720-8e7e-20678dc2fd71"
+session_id = "caa5dddc-9290-4e27-9f5e-575ba3598614"  # a BWM session with dual probe
 
 # Specify the revision of the pose estimation data
 # Setting to 'None' will use whatever the latest released revision is
 revision = None
 
-base_path = Path("E:/IBL")
+# base_path = Path("E:/IBL")
+base_path = Path.home() / "ibl_scratch"  # local directory
 base_path.mkdir(exist_ok=True)
 nwbfiles_folder_path = base_path / "nwbfiles"
 nwbfiles_folder_path.mkdir(exist_ok=True)

src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only_local_testing.py

@@ -3,109 +3,92 @@
 os.environ["JUPYTER_PLATFORM_DIRS"] = "1"  # Annoying
 
 import os
-
-# import traceback
-# from concurrent.futures import ProcessPoolExecutor, as_completed
 from pathlib import Path
-from shutil import rmtree
-
-# from tempfile import mkdtemp
-# from dandi.download import download as dandi_download
-# from dandi.organize import organize as dandi_organize
-# from dandi.upload import upload as dandi_upload
-# from neuroconv.tools.data_transfers import automatic_dandi_upload
-# from nwbinspector.tools import get_s3_urls_and_dandi_paths
+
 from one.api import ONE
 
-# from pynwb import NWBHDF5IO
-# from pynwb.image import ImageSeries
-# from tqdm import tqdm
-from ibl_to_nwb.brainwide_map import BrainwideMapConverter
-from ibl_to_nwb.brainwide_map.datainterfaces import (
-    BrainwideMapTrialsInterface,
-)
+from ibl_to_nwb.converters import BrainwideMapConverter
 from ibl_to_nwb.datainterfaces import (
+    BrainwideMapTrialsInterface,
     IblPoseEstimationInterface,
     IblSortingInterface,
     LickInterface,
     PupilTrackingInterface,
     RoiMotionEnergyInterface,
     WheelInterface,
 )
+from ibl_to_nwb.testing._consistency_checks import check_written_nwbfile_for_consistency
 
-base_path = Path.home() / "ibl_scratch"  # local directory
-# session = "d32876dd-8303-4720-8e7e-20678dc2fd71"
-session = "caa5dddc-9290-4e27-9f5e-575ba3598614"  # a BWM session with dual probe
+# select eid
+# -> run download_data_local first with this eid to set up the local folder structure and one cache
+eid = "caa5dddc-9290-4e27-9f5e-575ba3598614"
 
-nwbfile_path = base_path / "nwbfiles" / session / f"{session}.nwb"
-nwbfile_path.parent.mkdir(exist_ok=True)
+# folders
+base_path = Path.home() / "ibl_scratch"
+base_path.mkdir(exist_ok=True)
+nwbfiles_folder_path = base_path / "nwbfiles"
+nwbfiles_folder_path.mkdir(exist_ok=True)
 
 stub_test: bool = False
 cleanup: bool = False
 
 # assert len(os.environ.get("DANDI_API_KEY", "")) > 0, "Run `export DANDI_API_KEY=...`!"
+revision = None
 
-nwbfile_path.parent.mkdir(exist_ok=True)
-
-# Download behavior and spike sorted data for this session
-session_path = base_path / "ibl_conversion" / session
-cache_folder = base_path / "ibl_conversion" / session / "cache"
-session_one = ONE(
+# Initialize IBL (ONE) client to download processed data for this session
+one_cache_folder_path = base_path / "ibl_conversion" / eid / "cache"
+one = ONE(
     base_url="https://openalyx.internationalbrainlab.org",
     password="international",
-    silent=False,
-    cache_dir=cache_folder,
+    silent=True,
+    cache_dir=one_cache_folder_path,
 )
 
 # Initialize as many of each interface as we need across the streams
 data_interfaces = list()
 
 # These interfaces should always be present in source data
-data_interfaces.append(IblSortingInterface(session=session, cache_folder=cache_folder / "sorting"))
-data_interfaces.append(BrainwideMapTrialsInterface(one=session_one, session=session))
-data_interfaces.append(WheelInterface(one=session_one, session=session))
+data_interfaces.append(IblSortingInterface(one=one, session=eid, revision=revision))
+data_interfaces.append(BrainwideMapTrialsInterface(one=one, session=eid, revision=revision))
+data_interfaces.append(WheelInterface(one=one, session=eid, revision=revision))
 
 # These interfaces may not be present; check if they are before adding to list
-pose_estimation_files = session_one.list_datasets(eid=session, filename="*.dlc*")
+pose_estimation_files = one.list_datasets(eid=eid, filename="*.dlc*")
 for pose_estimation_file in pose_estimation_files:
     camera_name = pose_estimation_file.replace("alf/_ibl_", "").replace(".dlc.pqt", "")
-    data_interfaces.append(
-        IblPoseEstimationInterface(
-            one=session_one, session=session, camera_name=camera_name, include_pose=True, include_video=False
-        )
-    )
+    data_interfaces.append(IblPoseEstimationInterface(one=one, session=eid, camera_name=camera_name, revision=revision))
 
-pupil_tracking_files = session_one.list_datasets(eid=session, filename="*features*")
+pupil_tracking_files = one.list_datasets(eid=eid, filename="*features*")
 for pupil_tracking_file in pupil_tracking_files:
     camera_name = pupil_tracking_file.replace("alf/_ibl_", "").replace(".features.pqt", "")
-    data_interfaces.append(PupilTrackingInterface(one=session_one, session=session, camera_name=camera_name))
+    data_interfaces.append(PupilTrackingInterface(one=one, session=eid, camera_name=camera_name, revision=revision))
 
-roi_motion_energy_files = session_one.list_datasets(eid=session, filename="*ROIMotionEnergy.npy*")
+roi_motion_energy_files = one.list_datasets(eid=eid, filename="*ROIMotionEnergy.npy*")
 for roi_motion_energy_file in roi_motion_energy_files:
     camera_name = roi_motion_energy_file.replace("alf/", "").replace(".ROIMotionEnergy.npy", "")
-    data_interfaces.append(RoiMotionEnergyInterface(one=session_one, session=session, camera_name=camera_name))
+    data_interfaces.append(RoiMotionEnergyInterface(one=one, session=eid, camera_name=camera_name, revision=revision))
 
-if session_one.list_datasets(eid=session, collection="alf", filename="licks*"):
-    data_interfaces.append(LickInterface(one=session_one, session=session))
+if one.list_datasets(eid=eid, collection="alf", filename="licks*"):
+    data_interfaces.append(LickInterface(one=one, session=eid, revision=revision))
 
 # Run conversion
-session_converter = BrainwideMapConverter(
-    one=session_one, session=session, data_interfaces=data_interfaces, verbose=True
-)
+session_converter = BrainwideMapConverter(one=one, session=eid, data_interfaces=data_interfaces, verbose=True)
 
 metadata = session_converter.get_metadata()
-metadata["NWBFile"]["session_id"] = metadata["NWBFile"]["session_id"] + "-processed-only"
+subject_id = metadata["Subject"]["subject_id"]
+
+subject_folder_path = nwbfiles_folder_path / f"sub-{subject_id}"
+subject_folder_path.mkdir(exist_ok=True)
+nwbfile_path = subject_folder_path / f"sub-{subject_id}_ses-{eid}_desc-processed_.nwb"
 
 session_converter.run_conversion(
     nwbfile_path=nwbfile_path,
     metadata=metadata,
     overwrite=True,
 )
-# automatic_dandi_upload(
-#     dandiset_id="000409",
-#     nwb_folder_path=nwbfile_path.parent,
-#     cleanup=cleanup,
-# )
-if cleanup:
-    rmtree(cache_folder)
-    rmtree(nwbfile_path.parent)
+
+# if cleanup:
+#     rmtree(cache_folder)
+#     rmtree(nwbfile_path.parent)
+
+check_written_nwbfile_for_consistency(one=one, nwbfile_path=nwbfile_path)

src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only.py

@@ -5,13 +5,16 @@
 from ibl_to_nwb.converters import BrainwideMapConverter, IblSpikeGlxConverter
 from ibl_to_nwb.datainterfaces import RawVideoInterface
 
-session_id = "d32876dd-8303-4720-8e7e-20678dc2fd71"
+# session_id = "d32876dd-8303-4720-8e7e-20678dc2fd71"
+session_id = "caa5dddc-9290-4e27-9f5e-575ba3598614"  # a BWM session with dual probe
+
 
 # Specify the revision of the pose estimation data
 # Setting to 'None' will use whatever the latest released revision is
 revision = None
 
-base_path = Path("E:/IBL")
+# base_path = Path("E:/IBL")
+base_path = Path.home() / "ibl_scratch"  # local directory
 base_path.mkdir(exist_ok=True)
 nwbfiles_folder_path = base_path / "nwbfiles"
 nwbfiles_folder_path.mkdir(exist_ok=True)
@@ -28,7 +31,9 @@
 # Specify the path to the SpikeGLX files on the server but use ONE API for timestamps
 data_interfaces = []
 
-spikeglx_source_folder_path = Path("D:/example_data/ephy_testing_data/spikeglx/Noise4Sam_g0")
+# spikeglx_source_folder_path = Path("D:/example_data/ephy_testing_data/spikeglx/Noise4Sam_g0")
+session_folder = ibl_client.eid2path(session_id)
+spikeglx_source_folder_path = session_folder / "raw_ephys_data"
 spikeglx_subconverter = IblSpikeGlxConverter(folder_path=spikeglx_source_folder_path, one=ibl_client)
 data_interfaces.append(spikeglx_subconverter)
 

src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only_local_testing.py

@@ -0,0 +1,69 @@
+# %%
+from pathlib import Path
+
+from one.api import ONE
+
+from ibl_to_nwb.converters import BrainwideMapConverter, IblSpikeGlxConverter
+from ibl_to_nwb.datainterfaces import RawVideoInterface
+
+# select eid
+# -> run download_data_local first with this eid to set up the local folder structure and one cache
+eid = "caa5dddc-9290-4e27-9f5e-575ba3598614"
+
+# folders
+base_path = Path.home() / "ibl_scratch"
+base_path.mkdir(exist_ok=True)
+nwbfiles_folder_path = base_path / "nwbfiles"
+nwbfiles_folder_path.mkdir(exist_ok=True)
+
+# Initialize IBL (ONE) client to download processed data for this session
+one_cache_folder_path = base_path / "ibl_conversion" / eid / "cache"
+one = ONE(
+    base_url="https://openalyx.internationalbrainlab.org",
+    password="international",
+    silent=True,
+    cache_dir=one_cache_folder_path,
+)
+
+data_interfaces = []
+
+# %% ephys
+session_folder = one.eid2path(eid)
+spikeglx_source_folder_path = session_folder / "raw_ephys_data"
+
+# Specify the path to the SpikeGLX files on the server but use ONE API for timestamps
+spikeglx_subconverter = IblSpikeGlxConverter(folder_path=spikeglx_source_folder_path, one=one, eid=eid)
+data_interfaces.append(spikeglx_subconverter)
+
+# %% video
+# Raw video takes some special handling
+metadata_retrieval = BrainwideMapConverter(one=one, session=eid, data_interfaces=[], verbose=False)
+subject_id = metadata_retrieval.get_metadata()["Subject"]["subject_id"]
+
+pose_estimation_files = one.list_datasets(eid=eid, filename="*.dlc*")
+for pose_estimation_file in pose_estimation_files:
+    camera_name = pose_estimation_file.replace("alf/_ibl_", "").replace(".dlc.pqt", "")
+
+    video_interface = RawVideoInterface(
+        nwbfiles_folder_path=nwbfiles_folder_path,
+        subject_id=subject_id,
+        one=one,
+        session=eid,
+        camera_name=camera_name,
+    )
+    data_interfaces.append(video_interface)
+
+# Run conversion
+session_converter = BrainwideMapConverter(one=one, session=eid, data_interfaces=data_interfaces, verbose=False)
+
+metadata = session_converter.get_metadata()
+subject_id = metadata["Subject"]["subject_id"]
+
+subject_folder_path = nwbfiles_folder_path / f"sub-{subject_id}"
+subject_folder_path.mkdir(exist_ok=True)
+nwbfile_path = subject_folder_path / f"sub-{subject_id}_ses-{eid}_desc-raw_ecephys+raw_video_.nwb"
+
+session_converter.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True)
+
+# TODO: add some kind of raw-specific check
+# check_written_nwbfile_for_consistency(one=one, nwbfile_path=nwbfile_path)

src/ibl_to_nwb/_scripts/download_data.py

@@ -0,0 +1,45 @@
+# %%
+from pathlib import Path
+
+from one.api import ONE
+
+# %%
+eid = "caa5dddc-9290-4e27-9f5e-575ba3598614"  # a BWM eid with dual probe
+
+base_path = Path.home() / "ibl_scratch"  # local directory
+
+# Download behavior and spike sorted data for this eid
+session_path = base_path / "ibl_conversion" / eid
+cache_folder = base_path / "ibl_conversion" / eid / "cache"
+session_one = ONE(
+    base_url="https://openalyx.internationalbrainlab.org",
+    password="international",
+    silent=False,
+    cache_dir=cache_folder,
+)
+
+# %% latest revision
+revisions = session_one.list_revisions(eid)
+revision = revisions[-1]
+
+# %% list all datasets
+datasets = session_one.list_datasets(eid)
+
+# %% list all collections
+collections = session_one.list_collections(eid)
+
+# %%
+for dataset in datasets:
+    session_one.load_dataset(eid, dataset, download_only=True)
+
+# %% downloads all raw ephys data!
+collections = session_one.list_collections(eid, collection="raw_ephys_data/*")
+for collection in collections:
+    datasets = session_one.list_datasets(eid, collection=collection)
+    for dataset in datasets:
+        session_one.load_dataset(eid, dataset, download_only=True)
+
+# %% just the video data
+datasets = session_one.list_datasets(eid, collection="raw_video_data")
+for dataset in datasets:
+    session_one.load_dataset(eid, dataset, download_only=True)

src/ibl_to_nwb/converters/_brainwide_map_converter.py

@@ -2,7 +2,7 @@
 
 from neuroconv.utils import dict_deep_update, load_dict_from_file
 
-from src.ibl_to_nwb.converters._iblconverter import IblConverter
+from ibl_to_nwb.converters._iblconverter import IblConverter
 
 
 class BrainwideMapConverter(IblConverter):

src/ibl_to_nwb/converters/_ibl_spikeglx_converter.py

@@ -1,40 +1,36 @@
+import numpy as np
+from brainbox.io.one import EphysSessionLoader, SpikeSortingLoader
 from neuroconv.converters import SpikeGLXConverterPipe
 from one.api import ONE
 from pydantic import DirectoryPath
 from pynwb import NWBFile
 
 
 class IblSpikeGlxConverter(SpikeGLXConverterPipe):
-
-    def __init__(self, folder_path: DirectoryPath, one: ONE) -> None:
+    def __init__(self, folder_path: DirectoryPath, one: ONE, eid: str) -> None:
         super().__init__(folder_path=folder_path)
         self.one = one
+        self.eid = eid
 
     def temporally_align_data_interfaces(self) -> None:
         """Align the raw data timestamps to the other data streams using the ONE API."""
         # This is the syntax for aligning the raw timestamps; I cannot test this without the actual data as stored
         # on your end, so please work with Heberto if there are any problems after uncommenting
-        # probe_to_imec_map = {
-        #     "probe00": 0,
-        #     "probe01": 1,
-        # }
-        #
-        # ephys_session_loader = EphysSessionLoader(one=self.one, eid=session_id)
-        # probes = ephys_session_loader.probes
-        # for probe_name, pid in ephys_session_loader.probes.items():
-        #     spike_sorting_loader = SpikeSortingLoader(pid=pid, one=ibl_client)
-        #
-        #     probe_index = probe_to_imec_map[probe_name]
-        #     for band in ["ap", "lf"]:
-        #         recording_interface = next(
-        #             interface
-        #             for interface in self.data_interface_objects
-        #             if f"imec{probe_index}.{band}" in interface.source_data["file_path"]
-        #         )
-        #
-        #         band_info = spike_sorting_loader.raw_electrophysiology(band=band, stream=True)
-        #         aligned_timestamps = spike_sorting_loader.samples2times(numpy.arange(0, band_info.ns), direction='forward')
-        #         recording_interface.set_aligned_timestamps(aligned_timestamps=aligned_timestamps)
+        probe_to_imec_map = {
+            "probe00": 0,
+            "probe01": 1,
+        }
+
+        ephys_session_loader = EphysSessionLoader(one=self.one, eid=self.eid)
+        for probe_name, pid in ephys_session_loader.probes.items():
+            spike_sorting_loader = SpikeSortingLoader(pid=pid, one=self.one)
+
+            probe_index = probe_to_imec_map[probe_name]
+            for band in ["ap", "lf"]:
+                recording_interface = self.data_interface_objects[f"imec{probe_index}.{band}"]
+                sl = spike_sorting_loader.raw_electrophysiology(band=band, stream=True)
+                aligned_timestamps = spike_sorting_loader.samples2times(np.arange(0, sl.ns), direction="forward")
+                recording_interface.set_aligned_timestamps(aligned_timestamps=aligned_timestamps)
         pass
 
     def add_to_nwbfile(self, nwbfile: NWBFile, metadata) -> None: