Skip to content

Commit

Permalink
Merge pull request #3 from catalystneuro/add_time_synch
Browse files Browse the repository at this point in the history
Add time synch for Fox lab
  • Loading branch information
h-mayorquin authored Nov 6, 2024
2 parents 52a8cbf + 7751569 commit b51780d
Show file tree
Hide file tree
Showing 4 changed files with 397 additions and 27 deletions.
31 changes: 24 additions & 7 deletions src/fox_lab_to_nwb/behavior.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,6 @@ def add_to_nwbfile(
"PTrigger",
]

# TODO: figure how how to store this
# Synchronization signals
cam_sync = daq_struct["data"][:, 0]
cam_trigger = daq_struct["data"][:, 1]
opto_trigger = daq_struct["data"][:, 2]
ptrigger = daq_struct["data"][:, 8]

# Behavior signals
left_wing_beat_amplitude = daq_struct["data"][:, 3]
right_wing_beat_amplitude = daq_struct["data"][:, 4]
Expand Down Expand Up @@ -105,3 +98,27 @@ def add_to_nwbfile(

nwbfile.add_acquisition(lhutchen_time_series)
nwbfile.add_acquisition(rhutchen_time_series)

def extract_synchronization_signals_info(self):

mat = read_mat(self.file_path)
recording_structure = mat["rec"]
daq_struct = recording_structure["daq"]

daq_sampling_rate = daq_struct["fs"]

cam_sync = daq_struct["data"][:, 0]
cam_trigger = daq_struct["data"][:, 1]
opto_trigger = daq_struct["data"][:, 2]
ptrigger = daq_struct["data"][:, 8]

return_dict = {
"daq_sampling_rate": daq_sampling_rate,
"cam_sync": cam_sync,
"cam_trigger": cam_trigger,
"opto_trigger": opto_trigger,
"ptrigger": ptrigger,
}

return return_dict

207 changes: 207 additions & 0 deletions src/fox_lab_to_nwb/camera_utilites.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,207 @@
from lxml import etree
import os

def extract_phantom_metadata(xml_path):
# Parse XML using lxml
tree = etree.parse(xml_path)
root = tree.getroot()

# Create metadata dictionary
metadata = {}

# Define paths and their corresponding keys/types
paths = {
# Camera settings
"frame_rate": (".//FrameRateDouble", float),
"total_frames": (".//TotalImageCount", int),
"first_frame": (".//FirstMovieImage", int),
"image_count": (".//ImageCount", int),
# Image properties
"width": (".//biWidth", int),
"height": (".//biHeight", int),
"bit_depth": (".//biBitCount", int),
"bit_depth_recording": (".//RecBPP", int),
# Camera info
"camera_model": (".//CameraModel", str),
"camera_version": (".//CameraVersion", int),
"firmware_version": (".//FirmwareVersion", int),
"software_version": (".//SoftwareVersion", int),
"serial": (".//Serial", int),
# Timing
"shutter_ns": (".//ShutterNs", int),
"frame_delay_ns": (".//FrameDelayNs", int),
# Image settings
"compression": (".//Compression", int),
"saturation": (".//Saturation", float),
"brightness": (".//Bright", int),
"contrast": (".//Contrast", int),
"gamma": (".//Gamma", float),
# Trigger settings
"trigger_frame": (".//TrigFrame", int),
"post_trigger": (".//PostTrigger", int),
# Auto exposure
"auto_exposure": (".//AutoExposure", bool),
"auto_exp_level": (".//AutoExpLevel", int),
"auto_exp_speed": (".//AutoExpSpeed", int),
}

# Extract all metadata based on paths
for key, (xpath, type_conv) in paths.items():
element = root.find(xpath)
if element is not None and element.text:
try:
metadata[key] = type_conv(element.text)
except (ValueError, TypeError):
metadata[key] = None
else:
metadata[key] = None

# Special handling for trigger time
trigger_date = root.find(".//TriggerTime/Date")
trigger_time = root.find(".//TriggerTime/Time")
if trigger_date is not None and trigger_time is not None:
metadata["trigger_time"] = f"{trigger_date.text} {trigger_time.text}"

# Get image acquisition position and size
metadata["acquisition"] = {
"pos_x": int(root.find(".//ImPosXAcq").text) if root.find(".//ImPosXAcq") is not None else None,
"pos_y": int(root.find(".//ImPosYAcq").text) if root.find(".//ImPosYAcq") is not None else None,
"width": int(root.find(".//ImWidthAcq").text) if root.find(".//ImWidthAcq") is not None else None,
"height": int(root.find(".//ImHeightAcq").text) if root.find(".//ImHeightAcq") is not None else None,
}

# Get white balance gains
wb_element = root.find(".//WBGain")
if wb_element is not None:
metadata["white_balance"] = {
"red": float(wb_element.find("Red").text) if wb_element.find("Red") is not None else None,
"blue": float(wb_element.find("Blue").text) if wb_element.find("Blue") is not None else None,
}

return metadata


from typing import Any


def extract_fastec_metadata(file_path: str) -> dict[str, dict[str, Any]]:
"""
Extract metadata from a Fastec camera metadata file.
Parameters
----------
file_path : str
Path to the Fastec metadata file.
Returns
-------
Dict[str, Dict[str, Any]]
Nested dictionary containing the parsed metadata.
The top level dictionary has sections as keys ('image', 'camera', 'record', 'normalization').
Each section contains a dictionary of key-value pairs with automatically converted data types.
Notes
-----
The function automatically converts values to appropriate types:
- Integers for numeric values
- Floats for decimal numbers
- Lists for matrix values [x,y,z]
- Tuples for bit modes (e.g., "10:3")
- Strings for text and other values
Examples
--------
>>> metadata = extract_fastec_metadata('metadata.txt')
>>> frame_rate = metadata['record']['fps']
>>> resolution = (metadata['image']['width'], metadata['image']['height'])
Raises
------
FileNotFoundError
If the metadata file is not found.
PermissionError
If there are insufficient permissions to read the file.
"""

def parse_value(value: str) -> int | float | list[int] | tuple[int, int] | str:
"""
Parse a string value into its appropriate type.
Parameters
----------
value : str
The string value to parse
Returns
-------
int | float | list[int] | tuple[int, int] | str
Parsed value in its appropriate type
"""
# Try to convert to int
try:
return int(value)
except ValueError:
pass

# Try to convert to float
try:
return float(value)
except ValueError:
pass

# Handle matrix values [x,y,z]
if value.startswith("[") and value.endswith("]"):
try:
return [int(x) for x in value[1:-1].split(",")]
except ValueError:
return value

# Handle bit mode (e.g., "10:3")
if ":" in value and len(value.split(":")) == 2:
try:
return tuple(int(x) for x in value.split(":"))
except ValueError:
return value

# Return as string if no other type matches
return value

# Check if file exists
if not os.path.exists(file_path):
raise FileNotFoundError(f"Metadata file not found: {file_path}")

metadata: Dict[str, Dict[str, Any]] = {}
current_section: Union[str, None] = None

try:
with open(file_path, "r") as file:
for line in file:
line = line.strip()

# Skip empty lines
if not line:
continue

# Check if line is a section header
if line.startswith("[") and line.endswith("]"):
current_section = line[1:-1].lower()
metadata[current_section] = {}
continue

# Parse key-value pairs
if "=" in line and current_section is not None:
key, value = line.split("=", 1)
key = key.strip()
value = value.strip()

# Parse value to appropriate type
parsed_value = parse_value(value)

metadata[current_section][key] = parsed_value

return metadata

except PermissionError:
raise PermissionError(f"Insufficient permissions to read file: {file_path}")
except Exception as e:
raise Exception(f"Error parsing metadata file: {str(e)}")
86 changes: 82 additions & 4 deletions src/fox_lab_to_nwb/conversion_notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ https://www.biorxiv.org/content/10.1101/2024.03.13.583703v1

The data is available here:



## Trial structure

Folder name:
Expand Down Expand Up @@ -85,10 +83,10 @@ Here is an example output:
```

# Static TIFF files for confocal imaging
Not yet available
Those wont't be necessary

# Synchronization signal with Spike2
Not yet available
They are using another system to synchronize the data. This is not available.

# Intracellular electrophysiology data including the voltage trace and stimulus trace

Expand Down Expand Up @@ -131,6 +129,51 @@ bodyparts: L_ant, R_ant, L_base, R_base,

Sidecam does not have DLC analysis

The metadata as extracted from the txt files looks like this:

```
{'image': {'roi_x': 320,
'roi_y': 272,
'width': 640,
'height': 480,
'bit_mode': (10, 3),
'sensor_options': 'bin1x:subs1x',
'frame_count': 4351,
'trigger_frame': 4350,
'start_frame': 0,
'end_frame': 4350,
'time_stamp': '24:024:08:59:22.735044',
'comment': ''},
'camera': {'make': 'FASTEC',
'model': 'IL5SM8256',
'fpga_rev': '0x00020014',
'software_version': '2.5.3',
'mac_address': 'a4:1b:c0:00:05:3b',
'camera_name': 'SideCam',
'sensor_type': 'M5LA'},
'record': {'fps': 2000,
'shutter_speed': 100,
'multi_slope': (0, 0),
'trigger_setting': '100%',
'sync_in': '0x0',
'sync_out': '0x0'},
'normalization': {'red_balance': 4096,
'blue_balance': 4096,
'green_balance': 4096,
'brightness': 100,
'contrast': 100,
'gamma': 100,
'sensor_gain': 100,
'red_gain': 0.0,
'blue_gain': 0.0,
'green_gain': 0.0,
'red_matrix': [4096, 0, 0],
'blue_matrix': [0, 0, 4096],
'green_matrix': [0, 4096, 0],
'raw': 0,
'codec': 'MJPEG'}}
```

### Phantom
Files:
* XZ_1_186.mp4
Expand All @@ -141,6 +184,41 @@ This camera has an associated DLC analysis for the following body parts:

bodyparts: haltere

Example of metadata in the xml:

```xml

{'frame_rate': 4000.0,
'total_frames': 7242,
'first_frame': -7241,
'image_count': 7242,
'width': 512,
'height': 384,
'bit_depth': 8,
'bit_depth_recording': 12,
'camera_model': 'Phantom v7',
'camera_version': 7,
'firmware_version': 381,
'software_version': 804,
'serial': 6725,
'shutter_ns': 240000,
'frame_delay_ns': 0,
'compression': 0,
'saturation': -2.0,
'brightness': 49,
'contrast': -3,
'gamma': -1.0,
'trigger_frame': 0,
'post_trigger': 1,
'auto_exposure': True,
'auto_exp_level': 80,
'auto_exp_speed': 5,
'trigger_time': 'Mon Jul 13 1970 15:31:51.504 832',
'acquisition': {'pos_x': 0, 'pos_y': 0, 'width': 512, 'height': 384},
'white_balance': {'red': 1.0, 'blue': 1.0}}
```


### Photron

Files:
Expand Down
Loading

0 comments on commit b51780d

Please sign in to comment.