diff --git a/src/ibl_to_nwb/_scripts/convert_brainwide_map.py b/src/ibl_to_nwb/_scripts/convert_brainwide_map.py deleted file mode 100644 index 0250f5f..0000000 --- a/src/ibl_to_nwb/_scripts/convert_brainwide_map.py +++ /dev/null @@ -1,228 +0,0 @@ -# import os -# -# os.environ["JUPYTER_PLATFORM_DIRS"] = "1" # Annoying -# -# import os -# from pathlib import Path -# from shutil import rmtree -# -# # from neuroconv.tools.data_transfers import automatic_dandi_upload as neuroconv_automatic_dandi_upload -# from one.api import ONE -# -# from ibl_to_nwb.brainwide_map import BrainwideMapConverter -# from ibl_to_nwb.brainwide_map.datainterfaces import ( -# BrainwideMapTrialsInterface, -# ) -# from ibl_to_nwb.datainterfaces import ( -# IblPoseEstimationInterface, -# IblSortingInterface, -# IblStreamingApInterface, -# IblStreamingLfInterface, -# LickInterface, -# PupilTrackingInterface, -# RoiMotionEnergyInterface, -# WheelInterface, -# ) -# -# # def automatic_dandi_upload( -# # dandiset_id: str, -# # nwb_folder_path: str, -# # dandiset_folder_path: str = None, -# # version: str = "draft", -# # files_mode: str = "move", -# # staging: bool = False, -# # cleanup: bool = False, -# # ): -# # """ -# # Fully automated upload of NWBFiles to a DANDISet. -# # -# # Requires an API token set as an envrinment variable named DANDI_API_KEY. -# # -# # To set this in your bash terminal in Linux or MacOS, run -# # export DANDI_API_KEY=... -# # or in Windows -# # set DANDI_API_KEY=... -# # -# # DO NOT STORE THIS IN ANY PUBLICLY SHARED CODE. -# # -# # Parameters -# # ---------- -# # dandiset_id : str -# # Six-digit string identifier for the DANDISet the NWBFiles will be uploaded to. -# # nwb_folder_path : folder path -# # Folder containing the NWBFiles to be uploaded. -# # dandiset_folder_path : folder path, optional -# # A separate folder location within which to download the dandiset. -# # Used in cases where you do not have write permissions for the parent of the 'nwb_folder_path' directory. -# # Default behavior downloads the DANDISet to a folder adjacent to the 'nwb_folder_path'. -# # version : {None, "draft", "version"} -# # The default is "draft". -# # staging : bool, default: False -# # Is the DANDISet hosted on the staging server? This is mostly for testing purposes. -# # The default is False. -# # cleanup : bool, default: False -# # Whether to remove the dandiset folder path and nwb_folder_path. -# # Defaults to False. -# # """ -# # nwb_folder_path = Path(nwb_folder_path) -# # dandiset_folder_path = ( -# # Path(mkdtemp(dir=nwb_folder_path.parent)) if dandiset_folder_path is None else dandiset_folder_path -# # ) -# # dandiset_path = dandiset_folder_path / dandiset_id -# # assert os.getenv("DANDI_API_KEY"), ( -# # "Unable to find environment variable 'DANDI_API_KEY'. " -# # "Please retrieve your token from DANDI and set this environment variable." -# # ) -# # -# # url_base = "https://gui-staging.dandiarchive.org" if staging else "https://dandiarchive.org" -# # dandiset_url = f"{url_base}/dandiset/{dandiset_id}/{version}" -# # dandi_download(urls=dandiset_url, output_dir=str(dandiset_folder_path), get_metadata=True, get_assets=False) -# # assert dandiset_path.exists(), "DANDI download failed!" -# # -# # dandi_organize( -# # paths=str(nwb_folder_path), -# # dandiset_path=str(dandiset_path), -# # update_external_file_paths=True, -# # files_mode=files_mode, -# # media_files_mode=files_mode, -# # ) -# # organized_nwbfiles = dandiset_path.rglob("*.nwb") -# # -# # # DANDI has yet to implement forcing of session_id inclusion in organize step -# # # This manually enforces it when only a single session per subject is organized -# # for organized_nwbfile in organized_nwbfiles: -# # if "ses" not in organized_nwbfile.stem: -# # with NWBHDF5IO(path=organized_nwbfile, mode="r", load_namespaces=True) as io: -# # nwbfile = io.read() -# # session_id = nwbfile.session_id -# # dandi_stem = organized_nwbfile.stem -# # dandi_stem_split = dandi_stem.split("_") -# # dandi_stem_split.insert(1, f"ses-{session_id}") -# # corrected_name = "_".join(dandi_stem_split) + ".nwb" -# # organized_nwbfile.rename(organized_nwbfile.parent / corrected_name) -# # organized_nwbfiles = list(dandiset_path.rglob("*.nwb")) -# # # The above block can be removed once they add the feature -# # -# # # If any external images -# # image_folders = set(dandiset_path.rglob("*image*")) - set(organized_nwbfiles) -# # for image_folder in image_folders: -# # if "ses" not in image_folder.stem and len(organized_nwbfiles) == 1: # Think about multiple file case -# # corrected_name = "_".join(dandi_stem_split) -# # image_folder = image_folder.rename(image_folder.parent / corrected_name) -# # -# # # For image in folder, rename -# # with NWBHDF5IO(path=organized_nwbfiles[0], mode="r+", load_namespaces=True) as io: -# # nwbfile = io.read() -# # for _, object in nwbfile.objects.items(): -# # if isinstance(object, ImageSeries): -# # this_external_file = image_folder / Path(str(object.external_file[0])).name -# # corrected_name = "_".join(dandi_stem_split[:2]) + f"_{object.name}{this_external_file.suffix}" -# # this_external_file = this_external_file.rename(this_external_file.parent / corrected_name) -# # object.external_file[0] = "./" + str(this_external_file.relative_to(organized_nwbfile.parent)) -# # -# # assert len(list(dandiset_path.iterdir())) > 1, "DANDI organize failed!" -# # -# # dandi_instance = "dandi-staging" if staging else "dandi" -# # dandi_upload(paths=[dandiset_folder_path / dandiset_id], dandi_instance=dandi_instance) -# # -# # # Cleanup should be confirmed manually; Windows especially can complain -# # if cleanup: -# # try: -# # rmtree(path=dandiset_folder_path) -# # except PermissionError: # pragma: no cover -# # warn("Unable to clean up source files and dandiset! Please manually delete them.", stacklevel=2) -# -# -# base_path = Path("/home/jovyan/IBL") # prototype on DANDI Hub for now -# -# # session_retrieval_one = ONE( -# # base_url="https://openalyx.internationalbrainlab.org", password="international", silent=True -# # ) -# # brain_wide_sessions = session_retrieval_one.alyx.rest(url="sessions", action="list", tag="2022_Q4_IBL_et_al_BWM") -# -# # session = session_info["id"] -# session = "3e7ae7c0-fe8b-487c-9354-036236fa1010" -# -# nwbfile_path = base_path / "nwbfiles" / session / f"{session}.nwb" -# nwbfile_path.parent.mkdir(exist_ok=True) -# -# stub_test = False -# cleanup = False -# files_mode = "move" -# -# assert len(os.environ.get("DANDI_API_KEY", "")) > 0, "Run `export DANDI_API_KEY=...`!" -# -# # Download behavior and spike sorted data for this session -# session_path = base_path / "ibl_conversion" / session -# cache_folder = base_path / "ibl_conversion" / session / "cache" -# session_one = ONE( -# base_url="https://openalyx.internationalbrainlab.org", -# password="international", -# silent=True, -# cache_dir=cache_folder, -# ) -# -# # Get stream names from SI -# ap_stream_names = IblStreamingApInterface.get_stream_names(session=session) -# lf_stream_names = IblStreamingLfInterface.get_stream_names(session=session) -# -# # Initialize as many of each interface as we need across the streams -# data_interfaces = list() -# for stream_name in ap_stream_names: -# data_interfaces.append( -# IblStreamingApInterface(session=session, stream_name=stream_name, cache_folder=cache_folder / "ap_recordings") -# ) -# for stream_name in lf_stream_names: -# data_interfaces.append( -# IblStreamingLfInterface(session=session, stream_name=stream_name, cache_folder=cache_folder / "lf_recordings") -# ) -# -# # These interfaces should always be present in source data -# data_interfaces.append(IblSortingInterface(session=session, cache_folder=cache_folder / "sorting")) -# data_interfaces.append(BrainwideMapTrialsInterface(one=session_one, session=session)) -# data_interfaces.append(WheelInterface(one=session_one, session=session)) -# -# # These interfaces may not be present; check if they are before adding to list -# pose_estimation_files = session_one.list_datasets(eid=session, filename="*.dlc*") -# for pose_estimation_file in pose_estimation_files: -# camera_name = pose_estimation_file.replace("alf/_ibl_", "").replace(".dlc.pqt", "") -# data_interfaces.append( -# IblPoseEstimationInterface(one=session_one, session=session, camera_name=camera_name, include_video=True) -# ) -# -# pupil_tracking_files = session_one.list_datasets(eid=session, filename="*features*") -# for pupil_tracking_file in pupil_tracking_files: -# camera_name = pupil_tracking_file.replace("alf/_ibl_", "").replace(".features.pqt", "") -# data_interfaces.append(PupilTrackingInterface(one=session_one, session=session, camera_name=camera_name)) -# -# roi_motion_energy_files = session_one.list_datasets(eid=session, filename="*ROIMotionEnergy.npy*") -# for roi_motion_energy_file in roi_motion_energy_files: -# camera_name = roi_motion_energy_file.replace("alf/", "").replace(".ROIMotionEnergy.npy", "") -# data_interfaces.append(RoiMotionEnergyInterface(one=session_one, session=session, camera_name=camera_name)) -# -# if session_one.list_datasets(eid=session, collection="alf", filename="licks*"): -# data_interfaces.append(LickInterface(one=session_one, session=session)) -# -# # Run conversion -# session_converter = BrainwideMapConverter( -# one=session_one, session=session, data_interfaces=data_interfaces, verbose=False -# ) -# -# conversion_options = dict() -# if stub_test: -# for data_interface_name in session_converter.data_interface_objects: -# if "Ap" in data_interface_name or "Lf" in data_interface_name: -# conversion_options.update({data_interface_name: dict(stub_test=True)}) -# -# session_converter.run_conversion( -# nwbfile_path=nwbfile_path, -# metadata=session_converter.get_metadata(), -# conversion_options=conversion_options, -# overwrite=True, -# ) -# # automatic_dandi_upload( -# # dandiset_id="000409", nwb_folder_path=nwbfile_path.parent, cleanup=cleanup, files_mode=files_mode -# # ) -# if cleanup: -# rmtree(cache_folder) -# rmtree(nwbfile_path.parent) diff --git a/src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only.py b/src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only.py index 723e28f..af1ba5e 100644 --- a/src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only.py +++ b/src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only.py @@ -51,8 +51,6 @@ one=ibl_client, session=session_id, camera_name=camera_name, - include_video=False, - include_pose=True, revision=revision, ) ) diff --git a/src/ibl_to_nwb/datainterfaces/_lick_times.py b/src/ibl_to_nwb/datainterfaces/_lick_times.py index c65afa1..76f9a9e 100644 --- a/src/ibl_to_nwb/datainterfaces/_lick_times.py +++ b/src/ibl_to_nwb/datainterfaces/_lick_times.py @@ -29,5 +29,5 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): ], ) - behavior_module = get_module(nwbfile=nwbfile, name="behavior", description="Processed behavioral data.") - behavior_module.add(lick_events_table) + camera_module = get_module(nwbfile=nwbfile, name="camera", description="Processed camera data.") + camera_module.add(lick_events_table) diff --git a/src/ibl_to_nwb/datainterfaces/_pose_estimation.py b/src/ibl_to_nwb/datainterfaces/_pose_estimation.py index bbc9996..abf30d3 100644 --- a/src/ibl_to_nwb/datainterfaces/_pose_estimation.py +++ b/src/ibl_to_nwb/datainterfaces/_pose_estimation.py @@ -90,5 +90,6 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict) -> None: nodes=body_parts, ) pose_estimation_container = PoseEstimation(**pose_estimation_kwargs) - behavior_module = get_module(nwbfile=nwbfile, name="behavior", description="Processed behavioral data.") - behavior_module.add(pose_estimation_container) + + camera_module = get_module(nwbfile=nwbfile, name="camera", description="Processed camera data.") + camera_module.add(pose_estimation_container) diff --git a/src/ibl_to_nwb/datainterfaces/_pupil_tracking.py b/src/ibl_to_nwb/datainterfaces/_pupil_tracking.py index 1e38b0d..c307ef6 100644 --- a/src/ibl_to_nwb/datainterfaces/_pupil_tracking.py +++ b/src/ibl_to_nwb/datainterfaces/_pupil_tracking.py @@ -45,5 +45,5 @@ def add_to_nwbfile(self, nwbfile, metadata: dict): # in this case I'd say LeftPupilTracking and RightPupilTracking reads better pupil_tracking = PupilTracking(name=f"{left_or_right.capitalize()}PupilTracking", time_series=pupil_time_series) - behavior_module = get_module(nwbfile=nwbfile, name="behavior", description="Processed behavioral data.") - behavior_module.add(pupil_tracking) + camera_module = get_module(nwbfile=nwbfile, name="camera", description="Processed camera data.") + camera_module.add(pupil_tracking) diff --git a/src/ibl_to_nwb/datainterfaces/_roi_motion_energy.py b/src/ibl_to_nwb/datainterfaces/_roi_motion_energy.py index 2928610..4218647 100644 --- a/src/ibl_to_nwb/datainterfaces/_roi_motion_energy.py +++ b/src/ibl_to_nwb/datainterfaces/_roi_motion_energy.py @@ -38,5 +38,5 @@ def add_to_nwbfile(self, nwbfile, metadata: dict): unit="a.u.", ) - behavior_module = get_module(nwbfile=nwbfile, name="behavior", description="Processed behavioral data.") - behavior_module.add(motion_energy_series) + camera_module = get_module(nwbfile=nwbfile, name="camera", description="Processed camera data.") + camera_module.add(motion_energy_series) diff --git a/src/ibl_to_nwb/datainterfaces/_wheel_movement.py b/src/ibl_to_nwb/datainterfaces/_wheel_movement.py index a321b3d..234b307 100644 --- a/src/ibl_to_nwb/datainterfaces/_wheel_movement.py +++ b/src/ibl_to_nwb/datainterfaces/_wheel_movement.py @@ -78,7 +78,7 @@ def add_to_nwbfile(self, nwbfile, metadata: dict): unit="rad/s^2", ) - behavior_module = get_module(nwbfile=nwbfile, name="behavior", description="Processed behavioral data.") + behavior_module = get_module(nwbfile=nwbfile, name="wheel", description="Processed wheel data.") behavior_module.add(wheel_movement_intervals) behavior_module.add(compass_direction) behavior_module.add(velocity_series) diff --git a/src/ibl_to_nwb/testing/_consistency_checks.py b/src/ibl_to_nwb/testing/_consistency_checks.py index dc26fc0..ec5484a 100644 --- a/src/ibl_to_nwb/testing/_consistency_checks.py +++ b/src/ibl_to_nwb/testing/_consistency_checks.py @@ -27,9 +27,9 @@ def check_written_nwbfile_for_consistency(*, one: ONE, nwbfile_path: Path): def _check_wheel_data(*, eid: str, one: ONE, nwbfile: NWBFile, revision: str = None): - processing_module = nwbfile.processing["behavior"] + processing_module = nwbfile.processing["wheel"] wheel_position_series = processing_module.data_interfaces["CompassDirection"].spatial_series["WheelPositionSeries"] - wheel_movement_table = nwbfile.processing["behavior"].data_interfaces["WheelMovementIntervals"][:] + wheel_movement_table = processing_module.data_interfaces["WheelMovementIntervals"][:] # wheel position data_from_ONE = one.load_dataset(id=eid, dataset="_ibl_wheel.position", collection="alf") @@ -44,7 +44,7 @@ def _check_wheel_data(*, eid: str, one: ONE, nwbfile: NWBFile, revision: str = N # wheel movement intervals data_from_ONE = one.load_dataset(id=eid, dataset="_ibl_wheelMoves.intervals", collection="alf") data_from_NWB = wheel_movement_table[["start_time", "stop_time"]].values - assert_frame_equal(left=data_from_ONE, right=data_from_NWB) + assert_array_equal(x=data_from_ONE, y=data_from_NWB) # peak amplitude of wheel movement data_from_ONE = one.load_dataset(id=eid, dataset="_ibl_wheelMoves.peakAmplitude", collection="alf") @@ -53,7 +53,7 @@ def _check_wheel_data(*, eid: str, one: ONE, nwbfile: NWBFile, revision: str = N def _check_lick_data(*, eid: str, one: ONE, nwbfile: NWBFile): - processing_module = nwbfile.processing["behavior"] + processing_module = nwbfile.processing["camera"] lick_times_table = processing_module.data_interfaces["LickTimes"][:] data_from_NWB = lick_times_table["lick_time"].values @@ -62,7 +62,7 @@ def _check_lick_data(*, eid: str, one: ONE, nwbfile: NWBFile): def _check_roi_motion_energy_data(*, eid: str, one: ONE, nwbfile: NWBFile): - processing_module = nwbfile.processing["behavior"] + processing_module = nwbfile.processing["camera"] camera_views = ["body", "left", "right"] for view in camera_views: @@ -80,7 +80,7 @@ def _check_roi_motion_energy_data(*, eid: str, one: ONE, nwbfile: NWBFile): def _check_pose_estimation_data(*, eid: str, one: ONE, nwbfile: NWBFile, revision: str = None): - processing_module = nwbfile.processing["behavior"] + processing_module = nwbfile.processing["camera"] camera_views = ["body", "left", "right"] for view in camera_views: @@ -142,7 +142,7 @@ def _check_trials_data(*, eid: str, one: ONE, nwbfile: NWBFile): def _check_pupil_tracking_data(*, eid: str, one: ONE, nwbfile: NWBFile): - processing_module = nwbfile.processing["behavior"] + processing_module = nwbfile.processing["camera"] camera_views = ["left", "right"] for view in camera_views: @@ -165,7 +165,6 @@ def _check_pupil_tracking_data(*, eid: str, one: ONE, nwbfile: NWBFile): def _check_spike_sorting_data(*, eid: str, one: ONE, nwbfile: NWBFile, revision: str = None): - units_table = nwbfile.units[:] probe_names = units_table["probe_name"].unique()