From ad282753c9071fcacf42c3e21eefc71939fd6b91 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 5 Nov 2024 23:16:55 -0600 Subject: [PATCH 1/3] add camera synch --- .../assets/trialanalysis_photron.m | 29 +-- src/fox_lab_to_nwb/behavior.py | 31 ++- src/fox_lab_to_nwb/camera_utilites.py | 207 ++++++++++++++++++ src/fox_lab_to_nwb/conversion_notes.md | 86 +++++++- src/fox_lab_to_nwb/streets_conversion.py | 77 +++++-- 5 files changed, 376 insertions(+), 54 deletions(-) create mode 100644 src/fox_lab_to_nwb/camera_utilites.py diff --git a/src/fox_lab_to_nwb/assets/trialanalysis_photron.m b/src/fox_lab_to_nwb/assets/trialanalysis_photron.m index d2ba1aa..60f25ab 100644 --- a/src/fox_lab_to_nwb/assets/trialanalysis_photron.m +++ b/src/fox_lab_to_nwb/assets/trialanalysis_photron.m @@ -62,34 +62,7 @@ ptrig = ddata.daq.data(:,9); %photron trigger wtrig = ddata.daq.data(:,10); %wind trigger -%NOT CURRENTLY USING -%including in struct though so it's there if needed? -%otrig = ddata.daq.data(:,3); %opto trigger -dwbf = ddata.daq.data(:,6); -%wbf data from wingbeat amplifier needs to be multiplied by 100: -% 1V = 100Hz -dwbf = dwbf*100; %daq wingbeat freq, added 7/17 -dwbaL = ddata.daq.data(:,4); %temp -% twbaL = twbaL - mean(twbaL(1:2499)); %set to 0 for denoising -dwbaR = ddata.daq.data(:,5); -% twbaR = twbaR - mean(twbaR(1:2499)); -% hutchenL = ddata.daq.data(:,7); -% hutchenR = ddata.daq.data(:,8); - -%ALIGN CAMERAS TO DAQ - CHECK WITH MIKE AGAIN? -%find trigger times -ctrigtime = find(ctrig>3,1)/ddata.daq.fs; -ptrigtime = find(ptrig>3,1)/ddata.daq.fs; - -%topcam (currently skipping sidecam!) -meta_t = fastecMetaReader(fullfile(trial_path, 'TOPCAM_000000.txt')); -ts_t = linspace(1/meta_t.fs, meta_t.numframes/meta_t.fs, meta_t.numframes); -ts_t = ts_t-(ts_t(end)-ctrigtime); -%photron/phantom camera -mp = dir([trial_path filesep 'HALTCAM*.mii']); %since the number changes every time -meta_p = photronMetaReader(fullfile(mp.folder, mp.name)); -ts_p = linspace(1/meta_p.fs, meta_p.numframes/meta_p.fs, meta_p.numframes); -ts_p = ts_p-(ts_p(end)-ptrigtime); + q %% ANTENNAE %subtract height from y data because y = 0 is at top of image diff --git a/src/fox_lab_to_nwb/behavior.py b/src/fox_lab_to_nwb/behavior.py index bf973de..6872b74 100644 --- a/src/fox_lab_to_nwb/behavior.py +++ b/src/fox_lab_to_nwb/behavior.py @@ -38,13 +38,6 @@ def add_to_nwbfile( "PTrigger", ] - # TODO: figure how how to store this - # Synchronization signals - cam_sync = daq_struct["data"][:, 0] - cam_trigger = daq_struct["data"][:, 1] - opto_trigger = daq_struct["data"][:, 2] - ptrigger = daq_struct["data"][:, 8] - # Behavior signals left_wing_beat_amplitude = daq_struct["data"][:, 3] right_wing_beat_amplitude = daq_struct["data"][:, 4] @@ -105,3 +98,27 @@ def add_to_nwbfile( nwbfile.add_acquisition(lhutchen_time_series) nwbfile.add_acquisition(rhutchen_time_series) + + def extract_synchronization_signals_info(self): + + mat = read_mat(self.file_path) + recording_structure = mat["rec"] + daq_struct = recording_structure["daq"] + + daq_sampling_rate = daq_struct["fs"] + + cam_sync = daq_struct["data"][:, 0] + cam_trigger = daq_struct["data"][:, 1] + opto_trigger = daq_struct["data"][:, 2] + ptrigger = daq_struct["data"][:, 8] + + return_dict = { + "daq_sampling_rate": daq_sampling_rate, + "cam_sync": cam_sync, + "cam_trigger": cam_trigger, + "opto_trigger": opto_trigger, + "ptrigger": ptrigger, + } + + return return_dict + \ No newline at end of file diff --git a/src/fox_lab_to_nwb/camera_utilites.py b/src/fox_lab_to_nwb/camera_utilites.py new file mode 100644 index 0000000..7433824 --- /dev/null +++ b/src/fox_lab_to_nwb/camera_utilites.py @@ -0,0 +1,207 @@ +from lxml import etree +import os + +def extract_phantom_metadata(xml_path): + # Parse XML using lxml + tree = etree.parse(xml_path) + root = tree.getroot() + + # Create metadata dictionary + metadata = {} + + # Define paths and their corresponding keys/types + paths = { + # Camera settings + "frame_rate": (".//FrameRateDouble", float), + "total_frames": (".//TotalImageCount", int), + "first_frame": (".//FirstMovieImage", int), + "image_count": (".//ImageCount", int), + # Image properties + "width": (".//biWidth", int), + "height": (".//biHeight", int), + "bit_depth": (".//biBitCount", int), + "bit_depth_recording": (".//RecBPP", int), + # Camera info + "camera_model": (".//CameraModel", str), + "camera_version": (".//CameraVersion", int), + "firmware_version": (".//FirmwareVersion", int), + "software_version": (".//SoftwareVersion", int), + "serial": (".//Serial", int), + # Timing + "shutter_ns": (".//ShutterNs", int), + "frame_delay_ns": (".//FrameDelayNs", int), + # Image settings + "compression": (".//Compression", int), + "saturation": (".//Saturation", float), + "brightness": (".//Bright", int), + "contrast": (".//Contrast", int), + "gamma": (".//Gamma", float), + # Trigger settings + "trigger_frame": (".//TrigFrame", int), + "post_trigger": (".//PostTrigger", int), + # Auto exposure + "auto_exposure": (".//AutoExposure", bool), + "auto_exp_level": (".//AutoExpLevel", int), + "auto_exp_speed": (".//AutoExpSpeed", int), + } + + # Extract all metadata based on paths + for key, (xpath, type_conv) in paths.items(): + element = root.find(xpath) + if element is not None and element.text: + try: + metadata[key] = type_conv(element.text) + except (ValueError, TypeError): + metadata[key] = None + else: + metadata[key] = None + + # Special handling for trigger time + trigger_date = root.find(".//TriggerTime/Date") + trigger_time = root.find(".//TriggerTime/Time") + if trigger_date is not None and trigger_time is not None: + metadata["trigger_time"] = f"{trigger_date.text} {trigger_time.text}" + + # Get image acquisition position and size + metadata["acquisition"] = { + "pos_x": int(root.find(".//ImPosXAcq").text) if root.find(".//ImPosXAcq") is not None else None, + "pos_y": int(root.find(".//ImPosYAcq").text) if root.find(".//ImPosYAcq") is not None else None, + "width": int(root.find(".//ImWidthAcq").text) if root.find(".//ImWidthAcq") is not None else None, + "height": int(root.find(".//ImHeightAcq").text) if root.find(".//ImHeightAcq") is not None else None, + } + + # Get white balance gains + wb_element = root.find(".//WBGain") + if wb_element is not None: + metadata["white_balance"] = { + "red": float(wb_element.find("Red").text) if wb_element.find("Red") is not None else None, + "blue": float(wb_element.find("Blue").text) if wb_element.find("Blue") is not None else None, + } + + return metadata + + +from typing import Any + + +def extract_fastec_metadata(file_path: str) -> dict[str, dict[str, Any]]: + """ + Extract metadata from a Fastec camera metadata file. + + Parameters + ---------- + file_path : str + Path to the Fastec metadata file. + + Returns + ------- + Dict[str, Dict[str, Any]] + Nested dictionary containing the parsed metadata. + The top level dictionary has sections as keys ('image', 'camera', 'record', 'normalization'). + Each section contains a dictionary of key-value pairs with automatically converted data types. + + Notes + ----- + The function automatically converts values to appropriate types: + - Integers for numeric values + - Floats for decimal numbers + - Lists for matrix values [x,y,z] + - Tuples for bit modes (e.g., "10:3") + - Strings for text and other values + + Examples + -------- + >>> metadata = extract_fastec_metadata('metadata.txt') + >>> frame_rate = metadata['record']['fps'] + >>> resolution = (metadata['image']['width'], metadata['image']['height']) + + Raises + ------ + FileNotFoundError + If the metadata file is not found. + PermissionError + If there are insufficient permissions to read the file. + """ + + def parse_value(value: str) -> int | float | list[int] | tuple[int, int] | str: + """ + Parse a string value into its appropriate type. + + Parameters + ---------- + value : str + The string value to parse + + Returns + ------- + int | float | list[int] | tuple[int, int] | str + Parsed value in its appropriate type + """ + # Try to convert to int + try: + return int(value) + except ValueError: + pass + + # Try to convert to float + try: + return float(value) + except ValueError: + pass + + # Handle matrix values [x,y,z] + if value.startswith("[") and value.endswith("]"): + try: + return [int(x) for x in value[1:-1].split(",")] + except ValueError: + return value + + # Handle bit mode (e.g., "10:3") + if ":" in value and len(value.split(":")) == 2: + try: + return tuple(int(x) for x in value.split(":")) + except ValueError: + return value + + # Return as string if no other type matches + return value + + # Check if file exists + if not os.path.exists(file_path): + raise FileNotFoundError(f"Metadata file not found: {file_path}") + + metadata: Dict[str, Dict[str, Any]] = {} + current_section: Union[str, None] = None + + try: + with open(file_path, "r") as file: + for line in file: + line = line.strip() + + # Skip empty lines + if not line: + continue + + # Check if line is a section header + if line.startswith("[") and line.endswith("]"): + current_section = line[1:-1].lower() + metadata[current_section] = {} + continue + + # Parse key-value pairs + if "=" in line and current_section is not None: + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + + # Parse value to appropriate type + parsed_value = parse_value(value) + + metadata[current_section][key] = parsed_value + + return metadata + + except PermissionError: + raise PermissionError(f"Insufficient permissions to read file: {file_path}") + except Exception as e: + raise Exception(f"Error parsing metadata file: {str(e)}") diff --git a/src/fox_lab_to_nwb/conversion_notes.md b/src/fox_lab_to_nwb/conversion_notes.md index d857987..10d309d 100644 --- a/src/fox_lab_to_nwb/conversion_notes.md +++ b/src/fox_lab_to_nwb/conversion_notes.md @@ -10,8 +10,6 @@ https://www.biorxiv.org/content/10.1101/2024.03.13.583703v1 The data is available here: - - ## Trial structure Folder name: @@ -85,10 +83,10 @@ Here is an example output: ``` # Static TIFF files for confocal imaging -Not yet available +Those wont't be necessary # Synchronization signal with Spike2 -Not yet available +They are using another system to synchronize the data. This is not available. # Intracellular electrophysiology data including the voltage trace and stimulus trace @@ -131,6 +129,51 @@ bodyparts: L_ant, R_ant, L_base, R_base, Sidecam does not have DLC analysis +The metadata as extracted from the txt files looks like this: + +``` +{'image': {'roi_x': 320, + 'roi_y': 272, + 'width': 640, + 'height': 480, + 'bit_mode': (10, 3), + 'sensor_options': 'bin1x:subs1x', + 'frame_count': 4351, + 'trigger_frame': 4350, + 'start_frame': 0, + 'end_frame': 4350, + 'time_stamp': '24:024:08:59:22.735044', + 'comment': ''}, + 'camera': {'make': 'FASTEC', + 'model': 'IL5SM8256', + 'fpga_rev': '0x00020014', + 'software_version': '2.5.3', + 'mac_address': 'a4:1b:c0:00:05:3b', + 'camera_name': 'SideCam', + 'sensor_type': 'M5LA'}, + 'record': {'fps': 2000, + 'shutter_speed': 100, + 'multi_slope': (0, 0), + 'trigger_setting': '100%', + 'sync_in': '0x0', + 'sync_out': '0x0'}, + 'normalization': {'red_balance': 4096, + 'blue_balance': 4096, + 'green_balance': 4096, + 'brightness': 100, + 'contrast': 100, + 'gamma': 100, + 'sensor_gain': 100, + 'red_gain': 0.0, + 'blue_gain': 0.0, + 'green_gain': 0.0, + 'red_matrix': [4096, 0, 0], + 'blue_matrix': [0, 0, 4096], + 'green_matrix': [0, 4096, 0], + 'raw': 0, + 'codec': 'MJPEG'}} +``` + ### Phantom Files: * XZ_1_186.mp4 @@ -141,6 +184,41 @@ This camera has an associated DLC analysis for the following body parts: bodyparts: haltere +Example of metadata in the xml: + +```xml + +{'frame_rate': 4000.0, + 'total_frames': 7242, + 'first_frame': -7241, + 'image_count': 7242, + 'width': 512, + 'height': 384, + 'bit_depth': 8, + 'bit_depth_recording': 12, + 'camera_model': 'Phantom v7', + 'camera_version': 7, + 'firmware_version': 381, + 'software_version': 804, + 'serial': 6725, + 'shutter_ns': 240000, + 'frame_delay_ns': 0, + 'compression': 0, + 'saturation': -2.0, + 'brightness': 49, + 'contrast': -3, + 'gamma': -1.0, + 'trigger_frame': 0, + 'post_trigger': 1, + 'auto_exposure': True, + 'auto_exp_level': 80, + 'auto_exp_speed': 5, + 'trigger_time': 'Mon Jul 13 1970 15:31:51.504 832', + 'acquisition': {'pos_x': 0, 'pos_y': 0, 'width': 512, 'height': 384}, + 'white_balance': {'red': 1.0, 'blue': 1.0}} + ``` + + ### Photron Files: diff --git a/src/fox_lab_to_nwb/streets_conversion.py b/src/fox_lab_to_nwb/streets_conversion.py index e552ba2..fc83a51 100644 --- a/src/fox_lab_to_nwb/streets_conversion.py +++ b/src/fox_lab_to_nwb/streets_conversion.py @@ -8,8 +8,11 @@ from neuroconv.utils import dict_deep_update, load_dict_from_file from neuroconv import ConverterPipe from neuroconv.datainterfaces import DeepLabCutInterface, VideoInterface +import numpy as np from fox_lab_to_nwb.behavior import BehaviorInterface +from fox_lab_to_nwb.camera_utilites import extract_fastec_metadata, extract_phantom_metadata +from ndx_pose import PoseEstimation, PoseEstimationSeries # noqa: F401, this import is necessary for the conversion, until we fix things in neuroconv def run_trial_conversion(trial_folder_path: Path, output_dir_path: Optional[Path] = None, verbose: bool = True): @@ -37,30 +40,74 @@ def run_trial_conversion(trial_folder_path: Path, output_dir_path: Optional[Path nwbfile_path = output_dir_path / f"{session_id}.nwb" # Behavior interface - format = "fly2" # Authors said in email this might change + format = "fly2" # Authors said in email this might change, this is renamed matlab file daq_file_name = f"{cross}_{session_date}_{session_time}_f{fly_number}_r{trial_repeat_number}.{format}" file_path = trial_folder_path / daq_file_name behavior_interface = BehaviorInterface(file_path=file_path) - - # Video Interface - # TODO: are the names of the video files always the same per trial? we need - # More trials to find out - + + sync_info_dict = behavior_interface.extract_synchronization_signals_info() + + daq_sampling_rate = sync_info_dict["daq_sampling_rate"] + + ################# + # Video Interfaces + ################## side_cam_name = "SideCam_000000.avi" file_path = trial_folder_path / side_cam_name side_cam_interface = VideoInterface(file_paths=[file_path], metadata_key_name="SideCam") - + + fastec_metadata_file_path = trial_folder_path / f"{file_path.stem}.txt" + fastec_metadata = extract_fastec_metadata(fastec_metadata_file_path) + fastec_total_frames = fastec_metadata["image"]["frame_count"] + fastec_framerate = fastec_metadata["record"]["fps"] + + cam_trigger = sync_info_dict["cam_trigger"] + # Find trigger times (when signal goes above 3V) + fastec_trigger_index = np.where(cam_trigger > 3)[0][0] + fastec_trigger_time = fastec_trigger_index / daq_sampling_rate + + # Create timestamps for Fastec camera (TOPCAM) + fastec_timestamps = np.linspace(1/fastec_framerate, + fastec_total_frames/fastec_framerate, + fastec_total_frames) + # Align to trigger time + fastec_timestamps = fastec_timestamps - (fastec_timestamps[-1] - fastec_trigger_time) + + side_cam_interface.set_aligned_timestamps(aligned_timestamps=[fastec_timestamps]) + top_camera_name = "TOPCAM_000000.avi" file_path = trial_folder_path / top_camera_name - top_cam_interface = VideoInterface(file_paths=[file_path], metadata_key_name="TopCam") haltere_camera_name = "XZ_1_186.mp4" - file_path = trial_folder_path / haltere_camera_name - haltere_cam_interface = VideoInterface(file_paths=[file_path], metadata_key_name="BackCam") - - # DLC interface + haltere_cam_interface = VideoInterface(file_paths=[file_path], metadata_key_name="HaltereCam") + + phantom_metadata_file_path = trial_folder_path / "XZ_1_186.xml" + phantom_metadata = extract_phantom_metadata(phantom_metadata_file_path) + + phantom_total_frames = phantom_metadata["total_frames"] + phantom_framerate = phantom_metadata["frame_rate"] + + # Find trigger times (when signal goes above 3V) + ptrigger = sync_info_dict["ptrigger"] + phantom_trigger_index = np.where(ptrigger > 3)[0][0] + + phantom_trigger_time = phantom_trigger_index / daq_sampling_rate + + # Create timestamps for Phantom camera + phantom_timestamps = np.linspace(1/phantom_framerate, + phantom_total_frames/phantom_framerate, + phantom_total_frames) + # Align to trigger time + phantom_timestamps = phantom_timestamps - (phantom_timestamps[-1] - phantom_trigger_time) + + haltere_cam_interface.set_aligned_timestamps(aligned_timestamps=[phantom_timestamps]) + + ######################### + # DeepLabCut interfaces + ######################### + top_cam_dlc_file_name = "TOPCAM_000000DLC_resnet50_antennatrackingMar11shuffle1_100000.h5" top_cam_file_path = trial_folder_path / top_cam_dlc_file_name top_cam_dlc_interface = DeepLabCutInterface(file_path=top_cam_file_path) @@ -69,16 +116,16 @@ def run_trial_conversion(trial_folder_path: Path, output_dir_path: Optional[Path haltere_cam_file_path = trial_folder_path / haltere_cam_dlc_file_name haltere_cam_dlc_interface = DeepLabCutInterface(file_path=haltere_cam_file_path) - data_interface = { + data_interfaces = { "Behavior": behavior_interface, "SideCam": side_cam_interface, "TopCam": top_cam_interface, - "BackCam": haltere_cam_interface, + "HaltereCam": haltere_cam_interface, "DeepLabCutTopCam": top_cam_dlc_interface, "DeepLabCutHaltereCam": haltere_cam_dlc_interface, } - converter = ConverterPipe(data_interfaces=data_interface) + converter = ConverterPipe(data_interfaces=data_interfaces) # Add datetime to conversion metadata = converter.get_metadata() From f5c02b3146e0ce24a1ae8a0d646a97b8bbbb3746 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 5 Nov 2024 23:34:30 -0600 Subject: [PATCH 2/3] add top cam --- src/fox_lab_to_nwb/streets_conversion.py | 25 ++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/fox_lab_to_nwb/streets_conversion.py b/src/fox_lab_to_nwb/streets_conversion.py index fc83a51..3d96da6 100644 --- a/src/fox_lab_to_nwb/streets_conversion.py +++ b/src/fox_lab_to_nwb/streets_conversion.py @@ -75,10 +75,31 @@ def run_trial_conversion(trial_folder_path: Path, output_dir_path: Optional[Path side_cam_interface.set_aligned_timestamps(aligned_timestamps=[fastec_timestamps]) - top_camera_name = "TOPCAM_000000.avi" - file_path = trial_folder_path / top_camera_name + # For TopCam (similar to SideCam) + top_cam_name = "TOPCAM_000000.avi" + file_path = trial_folder_path / top_cam_name top_cam_interface = VideoInterface(file_paths=[file_path], metadata_key_name="TopCam") + fastec_metadata_file_path = trial_folder_path / f"{file_path.stem}.txt" + fastec_metadata = extract_fastec_metadata(fastec_metadata_file_path) + fastec_total_frames = fastec_metadata["image"]["frame_count"] + fastec_framerate = fastec_metadata["record"]["fps"] + + # Using the same camera trigger as SideCam since they use the same sync signal + cam_trigger = sync_info_dict["cam_trigger"] + fastec_trigger_index = np.where(cam_trigger > 3)[0][0] + fastec_trigger_time = fastec_trigger_index / daq_sampling_rate + + # Create timestamps for TopCam + fastec_timestamps = np.linspace(1/fastec_framerate, + fastec_total_frames/fastec_framerate, + fastec_total_frames) + # Align to trigger time + fastec_timestamps = fastec_timestamps - (fastec_timestamps[-1] - fastec_trigger_time) + + # Set the aligned timestamps for TopCam + top_cam_interface.set_aligned_timestamps(aligned_timestamps=[fastec_timestamps]) + haltere_camera_name = "XZ_1_186.mp4" file_path = trial_folder_path / haltere_camera_name haltere_cam_interface = VideoInterface(file_paths=[file_path], metadata_key_name="HaltereCam") From 77515691a055df1ccd2a7cee8b43e0e30bc6f40e Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 6 Nov 2024 12:14:47 -0600 Subject: [PATCH 3/3] ammend mistake --- .../assets/trialanalysis_photron.m | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/fox_lab_to_nwb/assets/trialanalysis_photron.m b/src/fox_lab_to_nwb/assets/trialanalysis_photron.m index 60f25ab..d2ba1aa 100644 --- a/src/fox_lab_to_nwb/assets/trialanalysis_photron.m +++ b/src/fox_lab_to_nwb/assets/trialanalysis_photron.m @@ -62,7 +62,34 @@ ptrig = ddata.daq.data(:,9); %photron trigger wtrig = ddata.daq.data(:,10); %wind trigger - q +%NOT CURRENTLY USING +%including in struct though so it's there if needed? +%otrig = ddata.daq.data(:,3); %opto trigger +dwbf = ddata.daq.data(:,6); +%wbf data from wingbeat amplifier needs to be multiplied by 100: +% 1V = 100Hz +dwbf = dwbf*100; %daq wingbeat freq, added 7/17 +dwbaL = ddata.daq.data(:,4); %temp +% twbaL = twbaL - mean(twbaL(1:2499)); %set to 0 for denoising +dwbaR = ddata.daq.data(:,5); +% twbaR = twbaR - mean(twbaR(1:2499)); +% hutchenL = ddata.daq.data(:,7); +% hutchenR = ddata.daq.data(:,8); + +%ALIGN CAMERAS TO DAQ - CHECK WITH MIKE AGAIN? +%find trigger times +ctrigtime = find(ctrig>3,1)/ddata.daq.fs; +ptrigtime = find(ptrig>3,1)/ddata.daq.fs; + +%topcam (currently skipping sidecam!) +meta_t = fastecMetaReader(fullfile(trial_path, 'TOPCAM_000000.txt')); +ts_t = linspace(1/meta_t.fs, meta_t.numframes/meta_t.fs, meta_t.numframes); +ts_t = ts_t-(ts_t(end)-ctrigtime); +%photron/phantom camera +mp = dir([trial_path filesep 'HALTCAM*.mii']); %since the number changes every time +meta_p = photronMetaReader(fullfile(mp.folder, mp.name)); +ts_p = linspace(1/meta_p.fs, meta_p.numframes/meta_p.fs, meta_p.numframes); +ts_p = ts_p-(ts_p(end)-ptrigtime); %% ANTENNAE %subtract height from y data because y = 0 is at top of image