output path related fixes / cleanups

src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only_local_testing.py

@@ -20,69 +20,75 @@
 )
 from ibl_to_nwb.testing._consistency_checks import check_written_nwbfile_for_consistency
 
-base_path = Path.home() / "ibl_scratch"  # local directory
-session = "caa5dddc-9290-4e27-9f5e-575ba3598614"  # a BWM session with dual probe
-nwbfile_path = base_path / "nwbfiles" / session / f"{session}.nwb"
-nwbfile_path.parent.mkdir(exist_ok=True, parents=True)
+# select eid
+# -> run download_data_local first with this eid to set up the local folder structure and one cache
+eid = "caa5dddc-9290-4e27-9f5e-575ba3598614"
+
+# folders
+base_path = Path.home() / "ibl_scratch"
+base_path.mkdir(exist_ok=True)
+nwbfiles_folder_path = base_path / "nwbfiles"
+nwbfiles_folder_path.mkdir(exist_ok=True)
 
 stub_test: bool = False
 cleanup: bool = False
 
 # assert len(os.environ.get("DANDI_API_KEY", "")) > 0, "Run `export DANDI_API_KEY=...`!"
 revision = None
 
-nwbfile_path.parent.mkdir(exist_ok=True)
-
-# Download behavior and spike sorted data for this session
-session_path = base_path / "ibl_conversion" / session
-cache_folder = base_path / "ibl_conversion" / session / "cache"
-session_one = ONE(
+# Initialize IBL (ONE) client to download processed data for this session
+one_cache_folder_path = base_path / "ibl_conversion" / eid / "cache"
+one = ONE(
     base_url="https://openalyx.internationalbrainlab.org",
     password="international",
-    silent=False,
-    cache_dir=cache_folder,
+    silent=True,
+    cache_dir=one_cache_folder_path,
 )
 
 # Initialize as many of each interface as we need across the streams
 data_interfaces = list()
 
 # These interfaces should always be present in source data
-data_interfaces.append(IblSortingInterface(session=session, cache_folder=cache_folder / "sorting", revision=revision))
-data_interfaces.append(BrainwideMapTrialsInterface(one=session_one, session=session, revision=revision))
-data_interfaces.append(WheelInterface(one=session_one, session=session, revision=revision))
+data_interfaces.append(IblSortingInterface(session=eid, cache_folder=one_cache_folder_path / "sorting", revision=revision))
+data_interfaces.append(BrainwideMapTrialsInterface(one=one, session=eid, revision=revision))
+data_interfaces.append(WheelInterface(one=one, session=eid, revision=revision))
 
 # These interfaces may not be present; check if they are before adding to list
-pose_estimation_files = session_one.list_datasets(eid=session, filename="*.dlc*")
+pose_estimation_files = one.list_datasets(eid=eid, filename="*.dlc*")
 for pose_estimation_file in pose_estimation_files:
     camera_name = pose_estimation_file.replace("alf/_ibl_", "").replace(".dlc.pqt", "")
     data_interfaces.append(
-        IblPoseEstimationInterface(one=session_one, session=session, camera_name=camera_name, revision=revision)
+        IblPoseEstimationInterface(one=one, session=eid, camera_name=camera_name, revision=revision)
     )
 
-pupil_tracking_files = session_one.list_datasets(eid=session, filename="*features*")
+pupil_tracking_files = one.list_datasets(eid=eid, filename="*features*")
 for pupil_tracking_file in pupil_tracking_files:
     camera_name = pupil_tracking_file.replace("alf/_ibl_", "").replace(".features.pqt", "")
     data_interfaces.append(
-        PupilTrackingInterface(one=session_one, session=session, camera_name=camera_name, revision=revision)
+        PupilTrackingInterface(one=one, session=eid, camera_name=camera_name, revision=revision)
     )
 
-roi_motion_energy_files = session_one.list_datasets(eid=session, filename="*ROIMotionEnergy.npy*")
+roi_motion_energy_files = one.list_datasets(eid=eid, filename="*ROIMotionEnergy.npy*")
 for roi_motion_energy_file in roi_motion_energy_files:
     camera_name = roi_motion_energy_file.replace("alf/", "").replace(".ROIMotionEnergy.npy", "")
     data_interfaces.append(
-        RoiMotionEnergyInterface(one=session_one, session=session, camera_name=camera_name, revision=revision)
+        RoiMotionEnergyInterface(one=one, session=eid, camera_name=camera_name, revision=revision)
     )
 
-if session_one.list_datasets(eid=session, collection="alf", filename="licks*"):
-    data_interfaces.append(LickInterface(one=session_one, session=session, revision=revision))
+if one.list_datasets(eid=eid, collection="alf", filename="licks*"):
+    data_interfaces.append(LickInterface(one=one, session=eid, revision=revision))
 
 # Run conversion
 session_converter = BrainwideMapConverter(
-    one=session_one, session=session, data_interfaces=data_interfaces, verbose=True
+    one=one, session=eid, data_interfaces=data_interfaces, verbose=True
 )
 
 metadata = session_converter.get_metadata()
-metadata["NWBFile"]["session_id"] = metadata["NWBFile"]["session_id"]  # + "-processed-only"
+subject_id = metadata["Subject"]["subject_id"]
+
+subject_folder_path = nwbfiles_folder_path / f"sub-{subject_id}"
+subject_folder_path.mkdir(exist_ok=True)
+nwbfile_path = subject_folder_path / f"sub-{subject_id}_ses-{eid}_desc-processed.nwb"
 
 session_converter.run_conversion(
     nwbfile_path=nwbfile_path,
@@ -94,8 +100,9 @@
 #     nwb_folder_path=nwbfile_path.parent,
 #     cleanup=cleanup,
 # )
-if cleanup:
-    rmtree(cache_folder)
-    rmtree(nwbfile_path.parent)
 
-check_written_nwbfile_for_consistency(one=session_one, nwbfile_path=nwbfile_path)
+# if cleanup:
+#     rmtree(cache_folder)
+#     rmtree(nwbfile_path.parent)
+
+check_written_nwbfile_for_consistency(one=one, nwbfile_path=nwbfile_path)

src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only_local_testing.py

@@ -57,12 +57,11 @@
 session_converter = BrainwideMapConverter(one=one, session=eid, data_interfaces=data_interfaces, verbose=False)
 
 metadata = session_converter.get_metadata()
-metadata["NWBFile"]["eid"] = metadata["NWBFile"]["eid"]
 subject_id = metadata["Subject"]["subject_id"]
 
 subject_folder_path = nwbfiles_folder_path / f"sub-{subject_id}"
 subject_folder_path.mkdir(exist_ok=True)
-nwbfile_path = subject_folder_path / f"sub-{subject_id}_ses-{eid}_desc-video.nwb"
+nwbfile_path = subject_folder_path / f"sub-{subject_id}_ses-{eid}_desc-raw.nwb"
 
 session_converter.run_conversion(
     nwbfile_path=nwbfile_path,

src/ibl_to_nwb/testing/_consistency_checks.py

@@ -176,7 +176,7 @@ def _check_spike_sorting_data(*, eid: str, one: ONE, nwbfile: NWBFile, revision:
     probe_names = units_table["probe_name"].unique()
 
     if revision is None:
-        revision = one.list_revisions(session)[-1]
+        revision = one.list_revisions(eid)[-1]
 
     spike_times = {}
     spike_clusters = {}