diff --git a/.github/workflows/build_env_run_tests.yml b/.github/workflows/build_env_run_tests.yml index 9372a4ca..b166c202 100644 --- a/.github/workflows/build_env_run_tests.yml +++ b/.github/workflows/build_env_run_tests.yml @@ -4,14 +4,14 @@ name: build_env_run_tests on: pull_request: - branches: [ main ] + branches: [main] types: [opened, reopened, synchronize] - workflow_dispatch: # allows running manually from Github's 'Actions' tab + workflow_dispatch: # allows running manually from Github's 'Actions' tab jobs: - build_env_pip_pyproject: # checks only for building env using pip and pyproject.toml - name: Build env using pip and pyproject.toml - runs-on: ubuntu-latest + build_env_run_tests: # checks for building env using pyproject.toml and runs codebase checks and tests + name: Build env using pip and pyproject.toml on ${{ matrix.os }} + runs-on: ${{ matrix.os }} if: github.event.pull_request.draft == false strategy: matrix: @@ -20,70 +20,43 @@ jobs: fail-fast: false defaults: run: - shell: bash -l {0} # reset shell for each step + shell: ${{ matrix.os == 'windows-latest' && 'cmd' || 'bash' }} -l {0} # Adjust shell based on OS steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Create venv and install dependencies run: | python -m venv .venv - source .venv/bin/activate + .venv/Scripts/activate || source .venv/bin/activate pip install -e .[dev] pip list - .venv/bin/python -c "import aeon" - - build_env_run_tests: # checks for building env using mamba and runs codebase checks and tests - name: Build env and run tests on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - if: github.event.pull_request.draft == false - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.11] - fail-fast: false - defaults: - run: - shell: bash -l {0} # reset shell for each step - steps: - - name: checkout repo - uses: actions/checkout@v2 - - name: set up conda env - uses: conda-incubator/setup-miniconda@v2 - with: - use-mamba: true - miniforge-variant: Mambaforge - python-version: ${{ matrix.python-version }} - environment-file: ./env_config/env.yml - activate-environment: aeon - - name: Update conda env with dev reqs - run: mamba env update -f ./env_config/env_dev.yml - - # Only run codebase checks and tests for ubuntu. + python -c "import aeon" + - name: Activate venv for later steps + run: | + echo "VIRTUAL_ENV=$(pwd)/.venv" >> $GITHUB_ENV + echo "$(pwd)/.venv/bin" >> $GITHUB_PATH # For Unix-like systems + echo "$(pwd)/.venv/Scripts" >> $GITHUB_PATH # For Windows + # Only run codebase checks and tests for Linux (ubuntu). - name: ruff - if: matrix.os == 'ubuntu-latest' - run: python -m ruff check --config ./pyproject.toml . + run: ruff check . - name: pyright - if: matrix.os == 'ubuntu-latest' - run: python -m pyright --level error --project ./pyproject.toml . + run: pyright --level error --project ./pyproject.toml . - name: pytest - if: matrix.os == 'ubuntu-latest' - run: python -m pytest tests/ - + run: pytest tests/ --ignore=tests/dj_pipeline - name: generate test coverage report - if: matrix.os == 'ubuntu-latest' + if: ${{ matrix.os == 'ubuntu-latest' }} run: | - python -m pytest --cov=aeon ./tests/ --cov-report=xml:./tests/test_coverage/test_coverage_report.xml - #python -m pytest --cov=aeon ./tests/ --cov-report=html:./tests/test_coverage/test_coverage_report_html + python -m pytest --cov=aeon tests/ --ignore=tests/dj_pipeline --cov-report=xml:tests/test_coverage/test_coverage_report.xml - name: upload test coverage report to codecov - if: matrix.os == 'ubuntu-latest' - uses: codecov/codecov-action@v2 + if: ${{ matrix.os == 'ubuntu-latest' }} + uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} - directory: ./tests/test_coverage/ + directory: tests/test_coverage/ files: test_coverage_report.xml fail_ci_if_error: true verbose: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 20d1ff13..21cbac48 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,4 @@ # For info on running pre-commit manually, see `pre-commit run --help` - -default_language_version: - python: python3.11 - files: "^(test|aeon)\/.*$" repos: - repo: meta @@ -10,7 +6,7 @@ repos: - id: identity - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: check-json - id: check-yaml @@ -25,20 +21,17 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - - repo: https://github.com/psf/black - rev: 23.7.0 - hooks: - - id: black - args: [--check, --config, ./pyproject.toml] - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.286 + rev: v0.6.4 hooks: + # Run the linter with the `--fix` flag. - id: ruff - args: [--config, ./pyproject.toml] + args: [ --fix ] + # Run the formatter. + - id: ruff-format - repo: https://github.com/RobertCraigie/pyright-python - rev: v1.1.324 + rev: v1.1.380 hooks: - id: pyright args: [--level, error, --project, ./pyproject.toml] diff --git a/aeon/README.md b/aeon/README.md index 4287ca86..792d6005 100644 --- a/aeon/README.md +++ b/aeon/README.md @@ -1 +1 @@ -# \ No newline at end of file +# diff --git a/aeon/__init__.py b/aeon/__init__.py index b59e77aa..2a691c53 100644 --- a/aeon/__init__.py +++ b/aeon/__init__.py @@ -9,5 +9,5 @@ finally: del version, PackageNotFoundError -# Set functions avaialable directly under the 'aeon' top-level namespace -from aeon.io.api import load +# Set functions available directly under the 'aeon' top-level namespace +from aeon.io.api import load as load # noqa: PLC0414 diff --git a/aeon/analysis/__init__.py b/aeon/analysis/__init__.py index 792d6005..e69de29b 100644 --- a/aeon/analysis/__init__.py +++ b/aeon/analysis/__init__.py @@ -1 +0,0 @@ -# diff --git a/aeon/analysis/block_plotting.py b/aeon/analysis/block_plotting.py index 027da966..67ebed32 100644 --- a/aeon/analysis/block_plotting.py +++ b/aeon/analysis/block_plotting.py @@ -1,17 +1,7 @@ -import os -import pathlib from colorsys import hls_to_rgb, rgb_to_hls -from contextlib import contextmanager -from pathlib import Path -import matplotlib.pyplot as plt import numpy as np -import pandas as pd import plotly -import plotly.express as px -import plotly.graph_objs as go -import seaborn as sns -from numpy.lib.stride_tricks import as_strided """Standardize subject colors, patch colors, and markers.""" @@ -35,27 +25,21 @@ "star", ] patch_markers_symbols = ["●", "⧓", "■", "⧗", "♦", "✖", "×", "▲", "★"] -patch_markers_dict = { - marker: symbol for marker, symbol in zip(patch_markers, patch_markers_symbols) -} +patch_markers_dict = dict(zip(patch_markers, patch_markers_symbols, strict=False)) patch_markers_linestyles = ["solid", "dash", "dot", "dashdot", "longdashdot"] def gen_hex_grad(hex_col, vals, min_l=0.3): """Generates an array of hex color values based on a gradient defined by unit-normalized values.""" # Convert hex to rgb to hls - h, l, s = rgb_to_hls( - *[int(hex_col.lstrip("#")[i: i + 2], 16) / 255 for i in (0, 2, 4)] - ) + h, l, s = rgb_to_hls(*[int(hex_col.lstrip("#")[i : i + 2], 16) / 255 for i in (0, 2, 4)]) # noqa: E741 grad = np.empty(shape=(len(vals),), dtype=" uuid.UUID: def fetch_stream(query, drop_pk=True): - """ + """Fetches data from a Stream table based on a query and returns it as a DataFrame. + Provided a query containing data from a Stream table, - fetch and aggregate the data into one DataFrame indexed by "time" + fetch and aggregate the data into one DataFrame indexed by "time" """ df = (query & "sample_count > 0").fetch(format="frame").reset_index() cols2explode = [ diff --git a/aeon/dj_pipeline/acquisition.py b/aeon/dj_pipeline/acquisition.py index b20c1a0c..f8048566 100644 --- a/aeon/dj_pipeline/acquisition.py +++ b/aeon/dj_pipeline/acquisition.py @@ -1,18 +1,18 @@ import datetime +import json import pathlib import re + import datajoint as dj import numpy as np import pandas as pd -import json -from aeon.io import api as io_api -from aeon.schema import schemas as aeon_schemas -from aeon.io import reader as io_reader from aeon.analysis import utils as analysis_utils - from aeon.dj_pipeline import get_schema_name, lab, subject from aeon.dj_pipeline.utils import paths +from aeon.io import api as io_api +from aeon.io import reader as io_reader +from aeon.schema import schemas as aeon_schemas logger = dj.logger schema = dj.schema(get_schema_name("acquisition")) @@ -181,7 +181,7 @@ class Epoch(dj.Manual): @classmethod def ingest_epochs(cls, experiment_name): - """Ingest epochs for the specified "experiment_name" """ + """Ingest epochs for the specified ``experiment_name``.""" device_name = _ref_device_mapping.get(experiment_name, "CameraTop") all_chunks, raw_data_dirs = _get_all_chunks(experiment_name, device_name) @@ -475,7 +475,7 @@ class MessageLog(dj.Part): -> master --- sample_count: int # number of data points acquired from this stream for a given chunk - timestamps: longblob # (datetime) + timestamps: longblob # (datetime) priority: longblob type: longblob message: longblob @@ -604,9 +604,7 @@ def _match_experiment_directory(experiment_name, path, directories): def create_chunk_restriction(experiment_name, start_time, end_time): - """ - Create a time restriction string for the chunks between the specified "start" and "end" times - """ + """Create a time restriction string for the chunks between the specified "start" and "end" times.""" start_restriction = f'"{start_time}" BETWEEN chunk_start AND chunk_end' end_restriction = f'"{end_time}" BETWEEN chunk_start AND chunk_end' start_query = Chunk & {"experiment_name": experiment_name} & start_restriction diff --git a/aeon/dj_pipeline/analysis/block_analysis.py b/aeon/dj_pipeline/analysis/block_analysis.py index ef50138f..b9baecdd 100644 --- a/aeon/dj_pipeline/analysis/block_analysis.py +++ b/aeon/dj_pipeline/analysis/block_analysis.py @@ -248,8 +248,7 @@ def make(self, key): ) # update block_end if last timestamp of encoder_df is before the current block_end - if encoder_df.index[-1] < block_end: - block_end = encoder_df.index[-1] + block_end = min(encoder_df.index[-1], block_end) # Subject data # Get all unique subjects that visited the environment over the entire exp; @@ -311,8 +310,7 @@ def make(self, key): ) # update block_end if last timestamp of pos_df is before the current block_end - if pos_df.index[-1] < block_end: - block_end = pos_df.index[-1] + block_end = min(pos_df.index[-1], block_end) if block_end != (Block & key).fetch1("block_end"): Block.update1({**key, "block_end": block_end}) @@ -474,21 +472,21 @@ def make(self, key): ) subject_in_patch = in_patch[subject_name] subject_in_patch_cum_time = subject_in_patch.cumsum().values * dt - all_subj_patch_pref_dict[patch["patch_name"]][subject_name][ - "cum_time" - ] = subject_in_patch_cum_time + all_subj_patch_pref_dict[patch["patch_name"]][subject_name]["cum_time"] = ( + subject_in_patch_cum_time + ) subj_pellets = closest_subjects_pellet_ts[closest_subjects_pellet_ts == subject_name] self.Patch.insert1( key - | dict( - patch_name=patch["patch_name"], - subject_name=subject_name, - in_patch_timestamps=subject_in_patch.index.values, - in_patch_time=subject_in_patch_cum_time[-1], - pellet_count=len(subj_pellets), - pellet_timestamps=subj_pellets.index.values, - wheel_cumsum_distance_travelled=cum_wheel_dist_subj_df[subject_name].values, - ) + | { + "patch_name": patch["patch_name"], + "subject_name": subject_name, + "in_patch_timestamps": subject_in_patch.index.values, + "in_patch_time": subject_in_patch_cum_time[-1], + "pellet_count": len(subj_pellets), + "pellet_timestamps": subj_pellets.index.values, + "wheel_cumsum_distance_travelled": cum_wheel_dist_subj_df[subject_name].values, + } ) # Now that we have computed all individual patch and subject values, we iterate again through @@ -515,20 +513,20 @@ def make(self, key): self.Preference.insert1( key - | dict( - patch_name=patch_name, - subject_name=subject_name, - cumulative_preference_by_time=cum_pref_time, - cumulative_preference_by_wheel=cum_pref_dist, - final_preference_by_time=cum_pref_time[-1], - final_preference_by_wheel=cum_pref_dist[-1], - ) + | { + "patch_name": patch_name, + "subject_name": subject_name, + "cumulative_preference_by_time": cum_pref_time, + "cumulative_preference_by_wheel": cum_pref_dist, + "final_preference_by_time": cum_pref_time[-1], + "final_preference_by_wheel": cum_pref_dist[-1], + } ) @schema class BlockPlots(dj.Computed): - definition = """ + definition = """ -> BlockAnalysis --- subject_positions_plot: longblob @@ -697,11 +695,11 @@ def make(self, key): x=wheel_ts, y=cum_pref, mode="lines", # + markers", - line=dict( - width=2, - color=subject_colors[subj_i], - dash=patch_markers_linestyles[patch_i], - ), + line={ + "width": 2, + "color": subject_colors[subj_i], + "dash": patch_markers_linestyles[patch_i], + }, name=f"{subj} - {p}: μ: {patch_mean}", ) ) @@ -719,13 +717,13 @@ def make(self, key): x=cur_cum_pel_ct["time"], y=cur_cum_pel_ct["cum_pref"], mode="markers", - marker=dict( - symbol=patch_markers[patch_i], - color=gen_hex_grad( + marker={ + "symbol": patch_markers[patch_i], + "color": gen_hex_grad( subject_colors[-1], cur_cum_pel_ct["norm_thresh_val"] ), - size=8, - ), + "size": 8, + }, showlegend=False, customdata=np.stack((cur_cum_pel_ct["threshold"],), axis=-1), hovertemplate="Threshold: %{customdata[0]:.2f} cm", @@ -737,7 +735,7 @@ def make(self, key): title=f"Cumulative Patch Preference - {title}", xaxis_title="Time", yaxis_title="Pref Index", - yaxis=dict(tickvals=np.arange(0, 1.1, 0.1)), + yaxis={"tickvals": np.arange(0, 1.1, 0.1)}, ) # Insert figures as json-formatted plotly plots diff --git a/aeon/dj_pipeline/create_experiments/device_type_mapper.json b/aeon/dj_pipeline/create_experiments/device_type_mapper.json index 848f0f3b..f9b16280 100644 --- a/aeon/dj_pipeline/create_experiments/device_type_mapper.json +++ b/aeon/dj_pipeline/create_experiments/device_type_mapper.json @@ -1 +1 @@ -{"VideoController": "CameraController", "CameraTop": "SpinnakerVideoSource", "CameraWest": "SpinnakerVideoSource", "CameraEast": "SpinnakerVideoSource", "CameraNorth": "SpinnakerVideoSource", "CameraSouth": "SpinnakerVideoSource", "CameraPatch1": "SpinnakerVideoSource", "CameraPatch2": "SpinnakerVideoSource", "CameraNest": "SpinnakerVideoSource", "AudioAmbient": "AudioSource", "Patch1": "UndergroundFeeder", "Patch2": "UndergroundFeeder", "WeightNest": "WeightScale", "TrackingTop": "PositionTracking", "ActivityCenter": "ActivityTracking", "ActivityArena": "ActivityTracking", "ActivityNest": "ActivityTracking", "ActivityPatch1": "ActivityTracking", "ActivityPatch2": "ActivityTracking", "InNest": "RegionTracking", "InPatch1": "RegionTracking", "InPatch2": "RegionTracking", "ArenaCenter": "DistanceFromPoint", "InArena": "InRange", "InCorridor": "InRange", "ClockSynchronizer": "TimestampGenerator", "Rfid": "Rfid Reader", "CameraPatch3": "SpinnakerVideoSource", "Patch3": "UndergroundFeeder", "Nest": "WeightScale", "RfidNest1": "RfidReader", "RfidNest2": "RfidReader", "RfidGate": "RfidReader", "RfidPatch1": "RfidReader", "RfidPatch2": "RfidReader", "RfidPatch3": "RfidReader", "LightCycle": "EnvironmentCondition"} \ No newline at end of file +{"VideoController": "CameraController", "CameraTop": "SpinnakerVideoSource", "CameraWest": "SpinnakerVideoSource", "CameraEast": "SpinnakerVideoSource", "CameraNorth": "SpinnakerVideoSource", "CameraSouth": "SpinnakerVideoSource", "CameraPatch1": "SpinnakerVideoSource", "CameraPatch2": "SpinnakerVideoSource", "CameraNest": "SpinnakerVideoSource", "AudioAmbient": "AudioSource", "Patch1": "UndergroundFeeder", "Patch2": "UndergroundFeeder", "WeightNest": "WeightScale", "TrackingTop": "PositionTracking", "ActivityCenter": "ActivityTracking", "ActivityArena": "ActivityTracking", "ActivityNest": "ActivityTracking", "ActivityPatch1": "ActivityTracking", "ActivityPatch2": "ActivityTracking", "InNest": "RegionTracking", "InPatch1": "RegionTracking", "InPatch2": "RegionTracking", "ArenaCenter": "DistanceFromPoint", "InArena": "InRange", "InCorridor": "InRange", "ClockSynchronizer": "TimestampGenerator", "Rfid": "Rfid Reader", "CameraPatch3": "SpinnakerVideoSource", "Patch3": "UndergroundFeeder", "Nest": "WeightScale", "RfidNest1": "RfidReader", "RfidNest2": "RfidReader", "RfidGate": "RfidReader", "RfidPatch1": "RfidReader", "RfidPatch2": "RfidReader", "RfidPatch3": "RfidReader", "LightCycle": "EnvironmentCondition"} diff --git a/aeon/dj_pipeline/create_experiments/insert_experiment_directory.ipynb b/aeon/dj_pipeline/create_experiments/insert_experiment_directory.ipynb index ee14ac1b..ee238780 100644 --- a/aeon/dj_pipeline/create_experiments/insert_experiment_directory.ipynb +++ b/aeon/dj_pipeline/create_experiments/insert_experiment_directory.ipynb @@ -301,12 +301,12 @@ "computer = \"AEON3\"\n", "\n", "acquisition.Experiment.Directory.insert1(\n", - " {\n", - " \"experiment_name\": experiment_name,\n", - " \"repository_name\": \"ceph_aeon\",\n", - " \"directory_type\": \"raw\",\n", - " \"directory_path\": f\"aeon/data/raw/{computer}/social0.1\"\n", - " },\n", + " {\n", + " \"experiment_name\": experiment_name,\n", + " \"repository_name\": \"ceph_aeon\",\n", + " \"directory_type\": \"raw\",\n", + " \"directory_path\": f\"aeon/data/raw/{computer}/social0.1\",\n", + " },\n", " skip_duplicates=True,\n", ")\n", "\n", @@ -315,12 +315,12 @@ "computer = \"AEON4\"\n", "\n", "acquisition.Experiment.Directory.insert1(\n", - " {\n", - " \"experiment_name\": experiment_name,\n", - " \"repository_name\": \"ceph_aeon\",\n", - " \"directory_type\": \"raw\",\n", - " \"directory_path\": f\"aeon/data/raw/{computer}/social0.1\"\n", - " },\n", + " {\n", + " \"experiment_name\": experiment_name,\n", + " \"repository_name\": \"ceph_aeon\",\n", + " \"directory_type\": \"raw\",\n", + " \"directory_path\": f\"aeon/data/raw/{computer}/social0.1\",\n", + " },\n", " skip_duplicates=True,\n", ")\n", "\n", diff --git a/aeon/dj_pipeline/create_experiments/setup_yml/Experiment0.1.yml b/aeon/dj_pipeline/create_experiments/setup_yml/Experiment0.1.yml index 66cde91a..2809393c 100644 --- a/aeon/dj_pipeline/create_experiments/setup_yml/Experiment0.1.yml +++ b/aeon/dj_pipeline/create_experiments/setup_yml/Experiment0.1.yml @@ -1,6 +1,6 @@ name: Experiment 0.1 start-time: 2021-06-02T23:49:41.0000000 -description: +description: data-path: /ceph/aeon/experiment0.1 time-bin-size: 1 arena: @@ -16,15 +16,15 @@ arena: gates: - position: *o0 clock-synchronizer: - serial-number: + serial-number: port-name: COM6 description: ClockSynchronizer ambient-microphone: - serial-number: + serial-number: description: AudioAmbient sample-rate: 192000 video-controller: - serial-number: + serial-number: port-name: COM3 description: VideoController standard-trigger-frequency: 50 @@ -71,14 +71,14 @@ cameras: trigger-source: HighSpeedTrigger gain: 10 patches: -- serial-number: +- serial-number: port-name: COM4 description: Patch1 position: *o0 radius: 4 starting-torque: 0 - workflow-path: -- serial-number: + workflow-path: +- serial-number: port-name: COM7 description: Patch2 position: *o0 @@ -89,4 +89,3 @@ weight-scales: - serial-number: SCALE1 description: WeightData nest: 1 - diff --git a/aeon/dj_pipeline/create_experiments/setup_yml/SocialExperiment0.yml b/aeon/dj_pipeline/create_experiments/setup_yml/SocialExperiment0.yml index 148eb180..d29f3c1e 100644 --- a/aeon/dj_pipeline/create_experiments/setup_yml/SocialExperiment0.yml +++ b/aeon/dj_pipeline/create_experiments/setup_yml/SocialExperiment0.yml @@ -1,6 +1,6 @@ name: Experiment 0.1 start-time: 2021-11-30 14:00:00 -description: +description: data-path: /ceph/preprocessed/socialexperiment0 time-bin-size: 1 arena: @@ -16,15 +16,15 @@ arena: gates: - position: *o0 clock-synchronizer: - serial-number: + serial-number: port-name: COM6 description: ClockSynchronizer ambient-microphone: - serial-number: + serial-number: description: AudioAmbient sample-rate: 192000 video-controller: - serial-number: + serial-number: port-name: COM3 description: VideoController standard-trigger-frequency: 50 @@ -71,14 +71,14 @@ cameras: trigger-source: HighSpeedTrigger gain: 10 patches: -- serial-number: +- serial-number: port-name: COM4 description: Patch1 position: *o0 radius: 4 starting-torque: 0 - workflow-path: -- serial-number: + workflow-path: +- serial-number: port-name: COM7 description: Patch2 position: *o0 @@ -89,4 +89,3 @@ weight-scales: - serial-number: SCALE1 description: WeightData nest: 1 - diff --git a/aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg b/aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg index 1addc0d8..94087629 100644 --- a/aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg +++ b/aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg @@ -4,7 +4,14 @@ VisitSummary - + VisitSummary @@ -13,7 +20,9 @@ VisitSubjectPosition - + VisitSubjectPosition @@ -22,7 +31,15 @@ VisitSubjectPosition.TimeSlice - + VisitSubjectPosition.TimeSlice @@ -36,7 +53,15 @@ VisitTimeDistribution - + VisitTimeDistribution @@ -45,7 +70,15 @@ ExperimentCamera - + ExperimentCamera @@ -54,7 +87,10 @@ CameraTracking - + CameraTracking @@ -68,7 +104,10 @@ Visit - + Visit @@ -92,7 +131,10 @@ Place - + Place @@ -106,7 +148,13 @@ Chunk - + Chunk @@ -125,7 +173,15 @@ Experiment - + Experiment @@ -144,7 +200,9 @@ Experiment.Subject - + Experiment.Subject @@ -161,4 +219,4 @@ - \ No newline at end of file + diff --git a/aeon/dj_pipeline/docs/datajoint_overview_diagram.svg b/aeon/dj_pipeline/docs/datajoint_overview_diagram.svg index 6cbe98cf..02590b4a 100644 --- a/aeon/dj_pipeline/docs/datajoint_overview_diagram.svg +++ b/aeon/dj_pipeline/docs/datajoint_overview_diagram.svg @@ -4,7 +4,14 @@ Arena - + Arena @@ -13,7 +20,15 @@ Experiment - + Experiment @@ -27,7 +42,14 @@ ExperimentFoodPatch - + ExperimentFoodPatch @@ -36,7 +58,13 @@ FoodPatchWheel - + FoodPatchWheel @@ -50,7 +78,9 @@ WheelState - + WheelState @@ -64,7 +94,13 @@ FoodPatchEvent - + FoodPatchEvent @@ -78,7 +114,8 @@ WeightScale - + WeightScale @@ -87,7 +124,14 @@ ExperimentWeightScale - + ExperimentWeightScale @@ -101,7 +145,8 @@ Camera - + Camera @@ -110,7 +155,15 @@ ExperimentCamera - + ExperimentCamera @@ -124,7 +177,9 @@ Epoch - + Epoch @@ -133,7 +188,13 @@ Chunk - + Chunk @@ -147,7 +208,8 @@ FoodPatch - + FoodPatch @@ -161,7 +223,10 @@ CameraTracking - + CameraTracking @@ -175,7 +240,19 @@ qc.CameraQC - + qc.CameraQC @@ -189,7 +266,15 @@ CameraTracking.Object - + CameraTracking.Object @@ -203,7 +288,10 @@ EventType - + EventType @@ -237,7 +325,13 @@ WeightMeasurement - + WeightMeasurement @@ -284,4 +378,4 @@ - \ No newline at end of file + diff --git a/aeon/dj_pipeline/docs/notebooks/analysis_diagram.svg b/aeon/dj_pipeline/docs/notebooks/analysis_diagram.svg index 1addc0d8..94087629 100644 --- a/aeon/dj_pipeline/docs/notebooks/analysis_diagram.svg +++ b/aeon/dj_pipeline/docs/notebooks/analysis_diagram.svg @@ -4,7 +4,14 @@ VisitSummary - + VisitSummary @@ -13,7 +20,9 @@ VisitSubjectPosition - + VisitSubjectPosition @@ -22,7 +31,15 @@ VisitSubjectPosition.TimeSlice - + VisitSubjectPosition.TimeSlice @@ -36,7 +53,15 @@ VisitTimeDistribution - + VisitTimeDistribution @@ -45,7 +70,15 @@ ExperimentCamera - + ExperimentCamera @@ -54,7 +87,10 @@ CameraTracking - + CameraTracking @@ -68,7 +104,10 @@ Visit - + Visit @@ -92,7 +131,10 @@ Place - + Place @@ -106,7 +148,13 @@ Chunk - + Chunk @@ -125,7 +173,15 @@ Experiment - + Experiment @@ -144,7 +200,9 @@ Experiment.Subject - + Experiment.Subject @@ -161,4 +219,4 @@ - \ No newline at end of file + diff --git a/aeon/dj_pipeline/docs/notebooks/diagram.ipynb b/aeon/dj_pipeline/docs/notebooks/diagram.ipynb index 179d25a7..fc6734d2 100644 --- a/aeon/dj_pipeline/docs/notebooks/diagram.ipynb +++ b/aeon/dj_pipeline/docs/notebooks/diagram.ipynb @@ -44,7 +44,7 @@ } ], "source": [ - "_db_prefix = 'aeon_'\n", + "_db_prefix = \"aeon_\"\n", "\n", "lab = dj.create_virtual_module(\"lab\", _db_prefix + \"lab\")\n", "subject = dj.create_virtual_module(\"subject\", _db_prefix + \"subject\")\n", @@ -488,7 +488,7 @@ " + acquisition.FoodPatchWheel\n", " + acquisition.FoodPatchEvent\n", " + acquisition.WheelState\n", - " + acquisition.EventType \n", + " + acquisition.EventType\n", " + acquisition.WeightMeasurement\n", " + qc.CameraQC\n", " + tracking.CameraTracking.Object\n", @@ -857,7 +857,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "datajoint_analysis_diagram.svg datajoint_overview_diagram.svg \u001B[0m\u001B[01;34mnotebooks\u001B[0m/\r\n" + "datajoint_analysis_diagram.svg datajoint_overview_diagram.svg \u001b[0m\u001b[01;34mnotebooks\u001b[0m/\r\n" ] } ], @@ -898,4 +898,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/aeon/dj_pipeline/docs/notebooks/diagram.svg b/aeon/dj_pipeline/docs/notebooks/diagram.svg index a0941dfb..d7872725 100644 --- a/aeon/dj_pipeline/docs/notebooks/diagram.svg +++ b/aeon/dj_pipeline/docs/notebooks/diagram.svg @@ -4,7 +4,10 @@ CameraTracking - + CameraTracking @@ -13,7 +16,15 @@ CameraTracking.Object - + CameraTracking.Object @@ -27,7 +38,8 @@ WeightScale - + WeightScale @@ -36,7 +48,14 @@ ExperimentWeightScale - + ExperimentWeightScale @@ -50,7 +69,9 @@ WheelState - + WheelState @@ -59,7 +80,14 @@ ExperimentFoodPatch - + ExperimentFoodPatch @@ -73,7 +101,13 @@ FoodPatchWheel - + FoodPatchWheel @@ -87,7 +121,13 @@ FoodPatchEvent - + FoodPatchEvent @@ -101,7 +141,13 @@ WeightMeasurement - + WeightMeasurement @@ -110,7 +156,14 @@ Arena - + Arena @@ -119,7 +172,15 @@ Experiment - + Experiment @@ -138,7 +199,10 @@ EventType - + EventType @@ -152,7 +216,15 @@ ExperimentCamera - + ExperimentCamera @@ -166,7 +238,19 @@ qc.CameraQC - + qc.CameraQC @@ -180,7 +264,8 @@ Camera - + Camera @@ -194,7 +279,9 @@ Epoch - + Epoch @@ -203,7 +290,13 @@ Chunk - + Chunk @@ -272,7 +365,8 @@ FoodPatch - + FoodPatch @@ -284,4 +378,4 @@ - \ No newline at end of file + diff --git a/aeon/dj_pipeline/docs/notebooks/social_experiments_block_analysis.ipynb b/aeon/dj_pipeline/docs/notebooks/social_experiments_block_analysis.ipynb index c7623089..02b0b919 100644 --- a/aeon/dj_pipeline/docs/notebooks/social_experiments_block_analysis.ipynb +++ b/aeon/dj_pipeline/docs/notebooks/social_experiments_block_analysis.ipynb @@ -28,7 +28,7 @@ "execution_count": null, "outputs": [], "source": [ - "analysis_vm = dj.create_virtual_module('aeon_block_analysis', 'aeon_block_analysis')" + "analysis_vm = dj.create_virtual_module(\"aeon_block_analysis\", \"aeon_block_analysis\")" ], "metadata": { "collapsed": false, @@ -83,7 +83,7 @@ "outputs": [], "source": [ "# Pick a block of interest\n", - "block_key = {'experiment_name': 'social0.1-aeon3', 'block_start': '2023-11-30 18:49:05.001984'}" + "block_key = {\"experiment_name\": \"social0.1-aeon3\", \"block_start\": \"2023-11-30 18:49:05.001984\"}" ], "metadata": { "collapsed": false, @@ -208,4 +208,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/aeon/dj_pipeline/populate/worker.py b/aeon/dj_pipeline/populate/worker.py index cbcdbb57..fc9968d1 100644 --- a/aeon/dj_pipeline/populate/worker.py +++ b/aeon/dj_pipeline/populate/worker.py @@ -44,13 +44,6 @@ def ingest_epochs_chunks(): acquisition.Chunk.ingest_chunks(experiment_name) -def ingest_environment_visits(): - """Extract and insert complete visits for experiments specified in AutomatedExperimentIngestion.""" - experiment_names = AutomatedExperimentIngestion.fetch("experiment_name") - # analysis.ingest_environment_visits(experiment_names) - pass - - # ---- Define worker(s) ---- # configure a worker to process `acquisition`-related tasks acquisition_worker = DataJointWorker( @@ -63,7 +56,6 @@ def ingest_environment_visits(): acquisition_worker(ingest_epochs_chunks) acquisition_worker(acquisition.EpochConfig) acquisition_worker(acquisition.Environment) -# acquisition_worker(ingest_environment_visits) acquisition_worker(block_analysis.BlockDetection) # configure a worker to handle pyrat sync @@ -111,6 +103,8 @@ def ingest_environment_visits(): analysis_worker(block_analysis.BlockSubjectAnalysis, max_calls=6) analysis_worker(block_analysis.BlockSubjectPlots, max_calls=6) + def get_workflow_operation_overview(): from datajoint_utilities.dj_worker.utils import get_workflow_operation_overview + return get_workflow_operation_overview(worker_schema_name=worker_schema_name, db_prefixes=[db_prefix]) diff --git a/aeon/dj_pipeline/qc.py b/aeon/dj_pipeline/qc.py index 0a9bd4e9..7044da0e 100644 --- a/aeon/dj_pipeline/qc.py +++ b/aeon/dj_pipeline/qc.py @@ -77,7 +77,7 @@ def make(self, key): "devices_schema_name" ), ) - stream_reader = getattr(getattr(devices_schema, device_name), "Video") + stream_reader = getattr(devices_schema, device_name).Video videodata = io_api.load( root=data_dirs, diff --git a/aeon/dj_pipeline/scripts/clone_and_freeze_exp01.py b/aeon/dj_pipeline/scripts/clone_and_freeze_exp01.py index 91e9e449..593bc7fe 100644 --- a/aeon/dj_pipeline/scripts/clone_and_freeze_exp01.py +++ b/aeon/dj_pipeline/scripts/clone_and_freeze_exp01.py @@ -1,6 +1,7 @@ """March 2022 Cloning and archiving schemas and data for experiment 0.1. """ + import os import datajoint as dj diff --git a/aeon/dj_pipeline/scripts/clone_and_freeze_exp02.py b/aeon/dj_pipeline/scripts/clone_and_freeze_exp02.py index ee1d7356..740932b2 100644 --- a/aeon/dj_pipeline/scripts/clone_and_freeze_exp02.py +++ b/aeon/dj_pipeline/scripts/clone_and_freeze_exp02.py @@ -1,10 +1,12 @@ -"""Jan 2024 -Cloning and archiving schemas and data for experiment 0.2. +"""Jan 2024: Cloning and archiving schemas and data for experiment 0.2. + The pipeline code associated with this archived data pipeline is here https://github.com/SainsburyWellcomeCentre/aeon_mecha/releases/tag/dj_exp02_stable """ -import os + import inspect +import os + import datajoint as dj from datajoint_utilities.dj_data_copy import db_migration from datajoint_utilities.dj_data_copy.pipeline_cloning import ClonedPipeline @@ -57,11 +59,11 @@ def data_copy(restriction, table_block_list, batch_size=None): def validate(): - """ - Validation of schemas migration - 1. for the provided list of schema names - validate all schemas have been migrated - 2. for each schema - validate all tables have been migrated - 3. for each table, validate all entries have been migrated + """Validates schemas migration. + + 1. for the provided list of schema names - validate all schemas have been migrated + 2. for each schema - validate all tables have been migrated + 3. for each table, validate all entries have been migrated """ missing_schemas = [] missing_tables = {} diff --git a/aeon/dj_pipeline/scripts/update_timestamps_longblob.py b/aeon/dj_pipeline/scripts/update_timestamps_longblob.py index 3980e6e8..4182c2e5 100644 --- a/aeon/dj_pipeline/scripts/update_timestamps_longblob.py +++ b/aeon/dj_pipeline/scripts/update_timestamps_longblob.py @@ -1,6 +1,7 @@ """July 2022 Upgrade all timestamps longblob fields with datajoint 0.13.7. """ + from datetime import datetime import datajoint as dj diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index 4cd482a0..e3d6ba12 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -14,7 +14,7 @@ schema = dj.Schema(get_schema_name("streams")) -@schema +@schema class StreamType(dj.Lookup): """Catalog of all steam types for the different device types used across Project Aeon. One StreamType corresponds to one reader class in `aeon.io.reader`. The combination of `stream_reader` and `stream_reader_kwargs` should fully specify the data loading routine for a particular device, using the `aeon.io.utils`.""" @@ -29,7 +29,7 @@ class StreamType(dj.Lookup): """ -@schema +@schema class DeviceType(dj.Lookup): """Catalog of all device types used across Project Aeon.""" @@ -46,7 +46,7 @@ class Stream(dj.Part): """ -@schema +@schema class Device(dj.Lookup): definition = """ # Physical devices, of a particular type, identified by unique serial number device_serial_number: varchar(12) @@ -55,7 +55,7 @@ class Device(dj.Lookup): """ -@schema +@schema class RfidReader(dj.Manual): definition = f""" # rfid_reader placement and operation for a particular time period, at a certain location, for a given experiment (auto-generated with aeon_mecha-unknown) @@ -82,7 +82,7 @@ class RemovalTime(dj.Part): """ -@schema +@schema class SpinnakerVideoSource(dj.Manual): definition = f""" # spinnaker_video_source placement and operation for a particular time period, at a certain location, for a given experiment (auto-generated with aeon_mecha-unknown) @@ -109,7 +109,7 @@ class RemovalTime(dj.Part): """ -@schema +@schema class UndergroundFeeder(dj.Manual): definition = f""" # underground_feeder placement and operation for a particular time period, at a certain location, for a given experiment (auto-generated with aeon_mecha-unknown) @@ -136,7 +136,7 @@ class RemovalTime(dj.Part): """ -@schema +@schema class WeightScale(dj.Manual): definition = f""" # weight_scale placement and operation for a particular time period, at a certain location, for a given experiment (auto-generated with aeon_mecha-unknown) @@ -163,7 +163,7 @@ class RemovalTime(dj.Part): """ -@schema +@schema class RfidReaderRfidEvents(dj.Imported): definition = """ # Raw per-chunk RfidEvents data stream from RfidReader (auto-generated with aeon_mecha-unknown) -> RfidReader @@ -838,5 +838,3 @@ def make(self, key): }, ignore_extra_fields=True, ) - - diff --git a/aeon/dj_pipeline/tracking.py b/aeon/dj_pipeline/tracking.py index 22ddf978..4f4d9de0 100644 --- a/aeon/dj_pipeline/tracking.py +++ b/aeon/dj_pipeline/tracking.py @@ -162,7 +162,7 @@ def make(self, key): "devices_schema_name" ), ) - stream_reader = getattr(getattr(devices_schema, device_name), "Pose") + stream_reader = getattr(devices_schema, device_name).Pose pose_data = io_api.load( root=data_dirs, diff --git a/aeon/dj_pipeline/utils/device_type_mapper.json b/aeon/dj_pipeline/utils/device_type_mapper.json index 7f041bd5..a28caebe 100644 --- a/aeon/dj_pipeline/utils/device_type_mapper.json +++ b/aeon/dj_pipeline/utils/device_type_mapper.json @@ -1 +1 @@ -{"ClockSynchronizer": "TimestampGenerator", "VideoController": "CameraController", "CameraTop": "SpinnakerVideoSource", "CameraWest": "SpinnakerVideoSource", "CameraEast": "SpinnakerVideoSource", "CameraNorth": "SpinnakerVideoSource", "CameraSouth": "SpinnakerVideoSource", "CameraNest": "SpinnakerVideoSource", "CameraPatch1": "SpinnakerVideoSource", "CameraPatch2": "SpinnakerVideoSource", "CameraPatch3": "SpinnakerVideoSource", "AudioAmbient": "AudioSource", "Patch1": "UndergroundFeeder", "Patch2": "UndergroundFeeder", "Patch3": "UndergroundFeeder", "Nest": "WeightScale", "RfidNest1": "RfidReader", "RfidNest2": "RfidReader", "RfidGate": "RfidReader", "RfidPatch1": "RfidReader", "RfidPatch2": "RfidReader", "RfidPatch3": "RfidReader", "LightCycle": "EnvironmentCondition"} \ No newline at end of file +{"ClockSynchronizer": "TimestampGenerator", "VideoController": "CameraController", "CameraTop": "SpinnakerVideoSource", "CameraWest": "SpinnakerVideoSource", "CameraEast": "SpinnakerVideoSource", "CameraNorth": "SpinnakerVideoSource", "CameraSouth": "SpinnakerVideoSource", "CameraNest": "SpinnakerVideoSource", "CameraPatch1": "SpinnakerVideoSource", "CameraPatch2": "SpinnakerVideoSource", "CameraPatch3": "SpinnakerVideoSource", "AudioAmbient": "AudioSource", "Patch1": "UndergroundFeeder", "Patch2": "UndergroundFeeder", "Patch3": "UndergroundFeeder", "Nest": "WeightScale", "RfidNest1": "RfidReader", "RfidNest2": "RfidReader", "RfidGate": "RfidReader", "RfidPatch1": "RfidReader", "RfidPatch2": "RfidReader", "RfidPatch3": "RfidReader", "LightCycle": "EnvironmentCondition"} diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index f2639c22..ce1c2775 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -44,8 +44,7 @@ def insert_stream_types(): def insert_device_types(devices_schema: DotMap, metadata_yml_filepath: Path): - """ - Use aeon.schema.schemas and metadata.yml to insert into streams.DeviceType and streams.Device. + """Use aeon.schema.schemas and metadata.yml to insert into streams.DeviceType and streams.Device. Only insert device types that were defined both in the device schema (e.g., exp02) and Metadata.yml. It then creates new device tables under streams schema. """ @@ -116,11 +115,12 @@ def insert_device_types(devices_schema: DotMap, metadata_yml_filepath: Path): streams.Device.insert(new_devices) -def extract_epoch_config(experiment_name: str, devices_schema, metadata_yml_filepath: str) -> dict: +def extract_epoch_config(experiment_name: str, devices_schema: DotMap, metadata_yml_filepath: str) -> dict: """Parse experiment metadata YAML file and extract epoch configuration. Args: experiment_name (str): Name of the experiment. + devices_schema (DotMap): DotMap object (e.g., exp02, octagon01) metadata_yml_filepath (str): path to the metadata YAML file. Returns: @@ -164,6 +164,7 @@ def extract_epoch_config(experiment_name: str, devices_schema, metadata_yml_file def ingest_epoch_metadata(experiment_name, devices_schema, metadata_yml_filepath): """Make entries into device tables.""" from aeon.dj_pipeline import acquisition + streams = dj.VirtualModule("streams", streams_maker.schema_name) if experiment_name.startswith("oct"): @@ -179,7 +180,7 @@ def ingest_epoch_metadata(experiment_name, devices_schema, metadata_yml_filepath epoch_start="MAX(epoch_start)", ) if len(acquisition.EpochConfig.Meta & previous_epoch) and epoch_config["commit"] == ( - acquisition.EpochConfig.Meta & previous_epoch + acquisition.EpochConfig.Meta & previous_epoch ).fetch1("commit"): # if identical commit -> no changes return diff --git a/aeon/dj_pipeline/utils/streams_maker.py b/aeon/dj_pipeline/utils/streams_maker.py index 3e5acafc..78e5ebaf 100644 --- a/aeon/dj_pipeline/utils/streams_maker.py +++ b/aeon/dj_pipeline/utils/streams_maker.py @@ -126,8 +126,8 @@ def get_device_stream_template(device_type: str, stream_type: str, streams_modul for col in stream.columns: if col.startswith("_"): continue - col = re.sub(r"\([^)]*\)", "", col) - table_definition += f"{col}: longblob\n " + new_col = re.sub(r"\([^)]*\)", "", col) + table_definition += f"{new_col}: longblob\n " class DeviceDataStream(dj.Imported): definition = table_definition diff --git a/aeon/dj_pipeline/webapps/sciviz/apk_requirements.txt b/aeon/dj_pipeline/webapps/sciviz/apk_requirements.txt index c9d2ebed..5ab05441 100644 --- a/aeon/dj_pipeline/webapps/sciviz/apk_requirements.txt +++ b/aeon/dj_pipeline/webapps/sciviz/apk_requirements.txt @@ -1,2 +1,2 @@ bash -git \ No newline at end of file +git diff --git a/aeon/io/__init__.py b/aeon/io/__init__.py index 792d6005..e69de29b 100644 --- a/aeon/io/__init__.py +++ b/aeon/io/__init__.py @@ -1 +0,0 @@ -# diff --git a/aeon/io/api.py b/aeon/io/api.py index d0ca0354..5d505ea6 100644 --- a/aeon/io/api.py +++ b/aeon/io/api.py @@ -61,11 +61,13 @@ def _empty(columns): def load(root, reader, start=None, end=None, time=None, tolerance=None, epoch=None, **kwargs): - """Extracts chunk data from the root path of an Aeon dataset using the specified data stream - reader. A subset of the data can be loaded by specifying an optional time range, or a list - of timestamps used to index the data on file. Returned data will be sorted chronologically. + """Extracts chunk data from the root path of an Aeon dataset. - :param str or PathLike root: The root path, or prioritised sequence of paths, where epoch data is stored. + Reads all chunk data using the specified data stream reader. A subset of the data can be loaded + by specifying an optional time range, or a list of timestamps used to index the data on file. + Returned data will be sorted chronologically. + + :param str or PathLike root: The root path, or prioritised sequence of paths, where data is stored. :param Reader reader: A data stream reader object used to read chunk data from the dataset. :param datetime, optional start: The left bound of the time range to extract. :param datetime, optional end: The right bound of the time range to extract. @@ -85,7 +87,7 @@ def load(root, reader, start=None, end=None, time=None, tolerance=None, epoch=No fileset = { chunk_key(fname): fname for path in root - for fname in path.glob(f"{epoch_pattern}/**/{reader.pattern}.{reader.extension}") + for fname in Path(path).glob(f"{epoch_pattern}/**/{reader.pattern}.{reader.extension}") } files = sorted(fileset.items()) @@ -101,7 +103,7 @@ def load(root, reader, start=None, end=None, time=None, tolerance=None, epoch=No filetimes = [chunk for (_, chunk), _ in files] files = [file for _, file in files] for key, values in time.groupby(by=chunk): - i = bisect.bisect_left(filetimes, key) + i = bisect.bisect_left(filetimes, key) # type: ignore if i < len(filetimes): frame = reader.read(files[i], **kwargs) _set_index(frame) @@ -144,10 +146,12 @@ def load(root, reader, start=None, end=None, time=None, tolerance=None, epoch=No import warnings if not data.index.has_duplicates: - warnings.warn(f"data index for {reader.pattern} contains out-of-order timestamps!") + warnings.warn( + f"data index for {reader.pattern} contains out-of-order timestamps!", stacklevel=2 + ) data = data.sort_index() else: - warnings.warn(f"data index for {reader.pattern} contains duplicate keys!") + warnings.warn(f"data index for {reader.pattern} contains duplicate keys!", stacklevel=2) data = data[~data.index.duplicated(keep="first")] return data.loc[start:end] return data diff --git a/aeon/io/device.py b/aeon/io/device.py index e68556ad..d7707fb0 100644 --- a/aeon/io/device.py +++ b/aeon/io/device.py @@ -1,4 +1,5 @@ import inspect + from typing_extensions import deprecated @@ -11,17 +12,17 @@ def compositeStream(pattern, *args): if inspect.isclass(binder_fn): for method in vars(binder_fn).values(): if isinstance(method, staticmethod): - registry.update(method.__func__(pattern)) + composite.update(method.__func__(pattern)) else: - registry.update(binder_fn(pattern)) - return registry + composite.update(binder_fn(pattern)) + return composite @deprecated("The Device class has been moved to the streams module.") class Device: """Groups multiple Readers into a logical device. - If a device contains a single stream reader with the same pattern as the device `name`, it will be + If a device contains a single stream reader with the same pattern as the device `name`, it will be considered a singleton, and the stream reader will be paired directly with the device without nesting. Attributes: @@ -33,7 +34,7 @@ class Device: def __init__(self, name, *args, pattern=None): self.name = name - self.registry = register(name if pattern is None else pattern, *args) + self.registry = compositeStream(name if pattern is None else pattern, *args) def __iter__(self): if len(self.registry) == 1: diff --git a/aeon/io/reader.py b/aeon/io/reader.py index 2f6f9460..abb6b97e 100644 --- a/aeon/io/reader.py +++ b/aeon/io/reader.py @@ -11,8 +11,8 @@ from dotmap import DotMap from aeon import util -from aeon.io.api import chunk, chunk_key from aeon.io.api import aeon as aeon_time +from aeon.io.api import chunk, chunk_key _SECONDS_PER_TICK = 32e-6 _payloadtypes = { @@ -43,7 +43,7 @@ def __init__(self, pattern, columns, extension): self.columns = columns self.extension = extension - def read(self, _): + def read(self, file): """Reads data from the specified file.""" return pd.DataFrame(columns=self.columns, index=pd.DatetimeIndex([])) @@ -94,7 +94,7 @@ def read(self, file): """Returns path and epoch information for the specified chunk.""" epoch, chunk = chunk_key(file) data = {"path": file, "epoch": epoch} - return pd.DataFrame(data, index=[chunk], columns=self.columns) + return pd.DataFrame(data, index=pd.Series(chunk), columns=self.columns) class Metadata(Reader): @@ -113,12 +113,13 @@ def read(self, file): workflow = metadata.pop("Workflow") commit = metadata.pop("Commit", pd.NA) data = {"workflow": workflow, "commit": commit, "metadata": [DotMap(metadata)]} - return pd.DataFrame(data, index=[time], columns=self.columns) + return pd.DataFrame(data, index=pd.Series(time), columns=self.columns) class Csv(Reader): - """Extracts data from comma-separated (csv) text files, where the first column - stores the Aeon timestamp, in seconds. + """Extracts data from comma-separated (CSV) text files. + + The first column stores the Aeon timestamp, in seconds. """ def __init__(self, pattern, columns, dtype=None, extension="csv"): @@ -187,8 +188,10 @@ def __init__(self, pattern): super().__init__(pattern, columns=["angle", "intensity"]) def read(self, file, downsample=True): - """Reads encoder data from the specified Harp binary file, and optionally downsamples - the frequency to 50Hz. + """Reads encoder data from the specified Harp binary file. + + By default the encoder data is downsampled to 50Hz. Setting downsample to + False or None can be used to force the raw data to be returned. """ data = super().read(file) if downsample is True: @@ -199,7 +202,7 @@ def read(self, file, downsample=True): if first_index is not None: # since data is absolute angular position we decimate by taking first of each bin chunk_origin = chunk(first_index) - data = data.resample('20ms', origin=chunk_origin).first() + data = data.resample("20ms", origin=chunk_origin).first() return data @@ -235,8 +238,9 @@ def __init__(self, pattern, value, tag): self.tag = tag def read(self, file): - """Reads a specific event code from digital data and matches it to the - specified unique identifier. + """Reads a specific event code from digital data. + + Each data value is matched against the unique event identifier. """ data = super().read(file) data = data[(data.event & self.value) == self.value] @@ -256,8 +260,9 @@ def __init__(self, pattern, mask, columns): self.mask = mask def read(self, file): - """Reads a specific event code from digital data and matches it to the - specified unique identifier. + """Reads a specific event code from digital data. + + Each data value is checked against the specified bitmask. """ data = super().read(file) state = data[self.columns] & self.mask @@ -316,15 +321,17 @@ def read(self, file: Path) -> pd.DataFrame: parts = self.get_bodyparts(config_file) # Using bodyparts, assign column names to Harp register values, and read data in default format. + BONSAI_SLEAP_V2 = 0.2 + BONSAI_SLEAP_V3 = 0.3 try: # Bonsai.Sleap0.2 - bonsai_sleap_v = 0.2 + bonsai_sleap_v = BONSAI_SLEAP_V2 columns = ["identity", "identity_likelihood"] for part in parts: columns.extend([f"{part}_x", f"{part}_y", f"{part}_likelihood"]) self.columns = columns data = super().read(file) except ValueError: # column mismatch; Bonsai.Sleap0.3 - bonsai_sleap_v = 0.3 + bonsai_sleap_v = BONSAI_SLEAP_V3 columns = ["identity"] columns.extend([f"{identity}_likelihood" for identity in identities]) for part in parts: @@ -346,13 +353,16 @@ def read(self, file: Path) -> pd.DataFrame: data = self.class_int2str(data, config_file) n_parts = len(parts) part_data_list = [pd.DataFrame()] * n_parts - new_columns = ["identity", "identity_likelihood", "part", "x", "y", "part_likelihood"] + new_columns = pd.Series(["identity", "identity_likelihood", "part", "x", "y", "part_likelihood"]) new_data = pd.DataFrame(columns=new_columns) for i, part in enumerate(parts): - part_columns = columns[0 : (len(identities) + 1)] if bonsai_sleap_v == 0.3 else columns[0:2] + part_columns = ( + columns[0 : (len(identities) + 1)] if bonsai_sleap_v == BONSAI_SLEAP_V3 else columns[0:2] + ) part_columns.extend([f"{part}_x", f"{part}_y", f"{part}_likelihood"]) part_data = pd.DataFrame(data[part_columns]) - if bonsai_sleap_v == 0.3: # combine all identity_likelihood cols into a single col as dict + if bonsai_sleap_v == BONSAI_SLEAP_V3: + # combine all identity_likelihood cols into a single col as dict part_data["identity_likelihood"] = part_data.apply( lambda row: {identity: row[f"{identity}_likelihood"] for identity in identities}, axis=1 ) @@ -369,17 +379,16 @@ def read(self, file: Path) -> pd.DataFrame: @staticmethod def get_class_names(config_file: Path) -> list[str]: """Returns a list of classes from a model's config file.""" - classes = None with open(config_file) as f: config = json.load(f) - if config_file.stem == "confmap_config": # SLEAP - try: - heads = config["model"]["heads"] - classes = util.find_nested_key(heads, "class_vectors")["classes"] - except KeyError as err: - if not classes: - raise KeyError(f"Cannot find class_vectors in {config_file}.") from err - return classes + if config_file.stem != "confmap_config": # SLEAP + raise ValueError(f"The model config file '{config_file}' is not supported.") + + try: + heads = config["model"]["heads"] + return util.find_nested_key(heads, "class_vectors")["classes"] + except KeyError as err: + raise KeyError(f"Cannot find class_vectors in {config_file}.") from err @staticmethod def get_bodyparts(config_file: Path) -> list[str]: @@ -428,6 +437,7 @@ def get_config_file(cls, config_file_dir: Path, config_file_names: None | list[s def from_dict(data, pattern=None): + """Converts a dictionary to a DotMap object.""" reader_type = data.get("type", None) if reader_type is not None: kwargs = {k: v for k, v in data.items() if k != "type"} @@ -439,6 +449,7 @@ def from_dict(data, pattern=None): def to_dict(dotmap): + """Converts a DotMap object to a dictionary.""" if isinstance(dotmap, Reader): kwargs = {k: v for k, v in vars(dotmap).items() if k not in ["pattern"] and not k.startswith("_")} kwargs["type"] = type(dotmap).__name__ diff --git a/aeon/io/video.py b/aeon/io/video.py index 79e43daa..26c49827 100644 --- a/aeon/io/video.py +++ b/aeon/io/video.py @@ -14,8 +14,8 @@ def frames(data): filename = None index = 0 try: - for frameidx, path in zip(data._frame, data._path): - if filename != path: + for frameidx, path in zip(data._frame, data._path, strict=False): + if filename != path or capture is None: if capture is not None: capture.release() capture = cv2.VideoCapture(path) @@ -49,7 +49,7 @@ def export(frames, file, fps, fourcc=None): for frame in frames: if writer is None: if fourcc is None: - fourcc = cv2.VideoWriter_fourcc("m", "p", "4", "v") + fourcc = cv2.VideoWriter_fourcc("m", "p", "4", "v") # type: ignore writer = cv2.VideoWriter(file, fourcc, fps, (frame.shape[1], frame.shape[0])) writer.write(frame) finally: diff --git a/aeon/qc/__init__.py b/aeon/qc/__init__.py deleted file mode 100644 index 792d6005..00000000 --- a/aeon/qc/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/aeon/qc/video.py b/aeon/qc/video.py deleted file mode 100644 index 1e090dc9..00000000 --- a/aeon/qc/video.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -from pathlib import Path - -import aeon.io.api as aeon - -root = "/ceph/aeon/test2/experiment0.1" -qcroot = "/ceph/aeon/aeon/qc/experiment0.1" -devicenames = [ - "FrameEast", - "FrameGate", - "FrameNorth", - "FramePatch1", - "FramePatch2", - "FrameSouth", - "FrameTop", - "FrameWest", -] - -for device in devicenames: - videochunks = aeon.chunkdata(root, device) - videochunks["epoch"] = videochunks.path.str.rsplit("/", n=3, expand=True)[1] - - stats = [] - frameshifts = [] - for key, period in videochunks.groupby(by="epoch"): - frame_offset = 0 - path = Path(os.path.join(qcroot, key, device)) - path.mkdir(parents=True, exist_ok=True) - for chunk in period.itertuples(): - outpath = Path(chunk.path.replace(root, qcroot)).with_suffix(".parquet") - print(f"[{key}] Analysing {device} {chunk.Index}... ", end="") - data = aeon.videoreader(chunk.path).reset_index() - deltas = data[data.columns[0:4]].diff() - deltas.columns = ["time_delta", "frame_delta", "hw_counter_delta", "hw_timestamp_delta"] - deltas["frame_offset"] = (deltas.hw_counter_delta - 1).cumsum() + frame_offset - drop_count = deltas.frame_offset.iloc[-1] - max_harp_delta = deltas.time_delta.max().total_seconds() - max_camera_delta = deltas.hw_timestamp_delta.max() / 1e9 # convert nanoseconds to seconds - print( - "drops: {} frameOffset: {} maxHarpDelta: {} s maxCameraDelta: {} s".format( - drop_count - frame_offset, drop_count, max_harp_delta, max_camera_delta - ) - ) - stats.append((drop_count, max_harp_delta, max_camera_delta, chunk.path)) - deltas.set_index(data.time, inplace=True) - deltas.to_parquet(outpath) - frameshifts.append(deltas) - frame_offset = drop_count diff --git a/aeon/schema/core.py b/aeon/schema/core.py index f3ca95a5..6f70c8b4 100644 --- a/aeon/schema/core.py +++ b/aeon/schema/core.py @@ -1,5 +1,5 @@ -from aeon.schema.streams import Stream, StreamGroup import aeon.io.reader as _reader +from aeon.schema.streams import Stream, StreamGroup class Heartbeat(Stream): diff --git a/aeon/schema/dataset.py b/aeon/schema/dataset.py index bbb7cbb8..0facd64f 100644 --- a/aeon/schema/dataset.py +++ b/aeon/schema/dataset.py @@ -1,8 +1,8 @@ from dotmap import DotMap import aeon.schema.core as stream -from aeon.schema.streams import Device from aeon.schema import foraging, octagon +from aeon.schema.streams import Device exp02 = DotMap( [ diff --git a/aeon/schema/foraging.py b/aeon/schema/foraging.py index 82865533..0eaf593c 100644 --- a/aeon/schema/foraging.py +++ b/aeon/schema/foraging.py @@ -1,5 +1,7 @@ from enum import Enum + import pandas as pd + import aeon.io.reader as _reader import aeon.schema.core as _stream from aeon.schema.streams import Stream, StreamGroup diff --git a/aeon/schema/schemas.py b/aeon/schema/schemas.py index 74618e7f..cea128d9 100644 --- a/aeon/schema/schemas.py +++ b/aeon/schema/schemas.py @@ -1,9 +1,8 @@ from dotmap import DotMap import aeon.schema.core as stream -from aeon.schema.streams import Device from aeon.schema import foraging, octagon, social_01, social_02, social_03 - +from aeon.schema.streams import Device exp02 = DotMap( [ diff --git a/aeon/schema/social_01.py b/aeon/schema/social_01.py index 4fee2d94..7f6e2ab0 100644 --- a/aeon/schema/social_01.py +++ b/aeon/schema/social_01.py @@ -3,7 +3,6 @@ class RfidEvents(Stream): - def __init__(self, path): path = path.replace("Rfid", "") if path.startswith("Events"): @@ -13,6 +12,5 @@ def __init__(self, path): class Pose(Stream): - def __init__(self, path): super().__init__(_reader.Pose(f"{path}_node-0*")) diff --git a/aeon/schema/social_02.py b/aeon/schema/social_02.py index 44c26c91..04946679 100644 --- a/aeon/schema/social_02.py +++ b/aeon/schema/social_02.py @@ -1,10 +1,9 @@ import aeon.io.reader as _reader -from aeon.schema.streams import Stream, StreamGroup from aeon.schema import core, foraging +from aeon.schema.streams import Stream, StreamGroup class Environment(StreamGroup): - def __init__(self, path): super().__init__(path) @@ -12,7 +11,9 @@ def __init__(self, path): class BlockState(Stream): def __init__(self, path): - super().__init__(_reader.Csv(f"{path}_BlockState_*", columns=["pellet_ct", "pellet_ct_thresh", "due_time"])) + super().__init__( + _reader.Csv(f"{path}_BlockState_*", columns=["pellet_ct", "pellet_ct_thresh", "due_time"]) + ) class LightEvents(Stream): def __init__(self, path): @@ -35,29 +36,29 @@ def __init__(self, path): class SubjectWeight(Stream): def __init__(self, path): - super().__init__(_reader.Csv(f"{path}_SubjectWeight_*", columns=["weight", "confidence", "subject_id", "int_id"])) + super().__init__( + _reader.Csv( + f"{path}_SubjectWeight_*", columns=["weight", "confidence", "subject_id", "int_id"] + ) + ) class Pose(Stream): - def __init__(self, path): super().__init__(_reader.Pose(f"{path}_test-node1*")) class WeightRaw(Stream): - def __init__(self, path): super().__init__(_reader.Harp(f"{path}_200_*", ["weight(g)", "stability"])) class WeightFiltered(Stream): - def __init__(self, path): super().__init__(_reader.Harp(f"{path}_202_*", ["weight(g)", "stability"])) class Patch(StreamGroup): - def __init__(self, path): super().__init__(path) @@ -83,6 +84,5 @@ def __init__(self, path): class RfidEvents(Stream): - def __init__(self, path): super().__init__(_reader.Harp(f"{path}_32*", ["rfid"])) diff --git a/aeon/schema/social_03.py b/aeon/schema/social_03.py index 558b39c9..18b05eec 100644 --- a/aeon/schema/social_03.py +++ b/aeon/schema/social_03.py @@ -3,6 +3,5 @@ class Pose(Stream): - def __init__(self, path): super().__init__(_reader.Pose(f"{path}_202_*")) diff --git a/aeon/schema/streams.py b/aeon/schema/streams.py index f306bf63..2c5d57b2 100644 --- a/aeon/schema/streams.py +++ b/aeon/schema/streams.py @@ -31,13 +31,12 @@ def __init__(self, path, *args): self._nested = ( member for member in vars(self.__class__).values() - if inspect.isclass(member) and issubclass(member, (Stream, StreamGroup)) + if inspect.isclass(member) and issubclass(member, Stream | StreamGroup) ) def __iter__(self): for factory in chain(self._nested, self._args): - for stream in iter(factory(self.path)): - yield stream + yield from iter(factory(self.path)) class Device: @@ -68,6 +67,7 @@ def _createStreams(path, args): warn( f"Stream group classes with default constructors are deprecated. {factory}", category=DeprecationWarning, + stacklevel=2, ) for method in vars(factory).values(): if isinstance(method, staticmethod): diff --git a/aeon/util.py b/aeon/util.py index a9dc88bc..2251eaad 100644 --- a/aeon/util.py +++ b/aeon/util.py @@ -1,4 +1,5 @@ """Utility functions.""" + from __future__ import annotations from typing import Any @@ -13,7 +14,7 @@ def find_nested_key(obj: dict | list, key: str) -> Any: found = find_nested_key(v, key) if found: return found - elif isinstance(obj, list): + else: for item in obj: found = find_nested_key(item, key) if found: diff --git a/docs/examples/dj_example_octagon1_experiment.ipynb b/docs/examples/dj_example_octagon1_experiment.ipynb index ae8e72f0..ca4cbf3a 100644 --- a/docs/examples/dj_example_octagon1_experiment.ipynb +++ b/docs/examples/dj_example_octagon1_experiment.ipynb @@ -48,9 +48,12 @@ "outputs": [], "source": [ "import datajoint as dj\n", - "dj.logger.setLevel('ERROR')\n", "\n", - "dj.config['custom']['database.prefix'] = 'aeon_test_' # data are ingested into schemas prefixed with \"aeon_test_\" for testing" + "dj.logger.setLevel(\"ERROR\")\n", + "\n", + "dj.config[\"custom\"][\n", + " \"database.prefix\"\n", + "] = \"aeon_test_\" # data are ingested into schemas prefixed with \"aeon_test_\" for testing" ] }, { @@ -85,8 +88,8 @@ "# then instead of importing the modules, you can use DataJoint's VirtualModule to access the pipeline\n", "# uncomment and run the codeblock below\n", "\n", - "#acquisition = dj.VirtualModule('acquisition', 'aeon_test_acquisition')\n", - "#streams = dj.VirtualModule('streams', 'aeon_test_streams')" + "# acquisition = dj.VirtualModule('acquisition', 'aeon_test_acquisition')\n", + "# streams = dj.VirtualModule('streams', 'aeon_test_streams')" ] }, { @@ -2665,7 +2668,7 @@ "metadata": {}, "outputs": [], "source": [ - "exp_key = {'experiment_name': 'oct1.0-r0'}" + "exp_key = {\"experiment_name\": \"oct1.0-r0\"}" ] }, { @@ -4034,7 +4037,7 @@ } ], "source": [ - "df_oscpoke = (streams.OSCPoke & 'chunk_start BETWEEN \"2022-08-01\" AND \"2022-08-03\"').fetch(format='frame')\n", + "df_oscpoke = (streams.OSCPoke & 'chunk_start BETWEEN \"2022-08-01\" AND \"2022-08-03\"').fetch(format=\"frame\")\n", "df_oscpoke" ] }, @@ -4371,7 +4374,9 @@ } ], "source": [ - "df_oscpoke.explode(['timestamps', 'typetag', 'wall_id', 'poke_id', 'reward', 'reward_interval', 'delay', 'led_delay'])" + "df_oscpoke.explode(\n", + " [\"timestamps\", \"typetag\", \"wall_id\", \"poke_id\", \"reward\", \"reward_interval\", \"delay\", \"led_delay\"]\n", + ")" ] }, { @@ -5050,9 +5055,11 @@ } ], "source": [ - "(streams.WallBeamBreak0 * streams.ExperimentWall\n", - " & 'wall_name = \"Wall4\"'\n", - " & 'chunk_start BETWEEN \"2022-08-01\" AND \"2022-08-03\"')" + "(\n", + " streams.WallBeamBreak0 * streams.ExperimentWall\n", + " & 'wall_name = \"Wall4\"'\n", + " & 'chunk_start BETWEEN \"2022-08-01\" AND \"2022-08-03\"'\n", + ")" ] }, { @@ -5195,9 +5202,11 @@ } ], "source": [ - "df_wall4_beambreak0 = (streams.WallBeamBreak0 * streams.ExperimentWall\n", - " & 'wall_name = \"Wall4\"'\n", - " & 'chunk_start BETWEEN \"2022-08-01\" AND \"2022-08-03\"').fetch(format='frame')\n", + "df_wall4_beambreak0 = (\n", + " streams.WallBeamBreak0 * streams.ExperimentWall\n", + " & 'wall_name = \"Wall4\"'\n", + " & 'chunk_start BETWEEN \"2022-08-01\" AND \"2022-08-03\"'\n", + ").fetch(format=\"frame\")\n", "df_wall4_beambreak0" ] }, @@ -5399,7 +5408,7 @@ } ], "source": [ - "df_wall4_beambreak0.explode(['timestamps', 'state'])" + "df_wall4_beambreak0.explode([\"timestamps\", \"state\"])" ] }, { diff --git a/docs/examples/understanding_aeon_data_architecture.ipynb b/docs/examples/understanding_aeon_data_architecture.ipynb index e3df6981..ee491c40 100644 --- a/docs/examples/understanding_aeon_data_architecture.ipynb +++ b/docs/examples/understanding_aeon_data_architecture.ipynb @@ -129,7 +129,7 @@ "time_set = pd.concat(\n", " [\n", " pd.Series(pd.date_range(start_time, start_time + pd.Timedelta(hours=1), freq=\"1s\")),\n", - " pd.Series(pd.date_range(end_time, end_time + pd.Timedelta(hours=1), freq=\"1s\"))\n", + " pd.Series(pd.date_range(end_time, end_time + pd.Timedelta(hours=1), freq=\"1s\")),\n", " ]\n", ")" ] @@ -143,15 +143,16 @@ "\"\"\"Creating a new `Reader` class\"\"\"\n", "\n", "# All readers are subclassed from the base `Reader` class. They thus all contain a `read` method,\n", - "# for returning data from a file in the form of a pandas DataFrame, and the following attributes, \n", + "# for returning data from a file in the form of a pandas DataFrame, and the following attributes,\n", "# which must be specified on object construction:\n", "# `pattern`: a prefix in the filename used by `aeon.io.api.load` to find matching files to load\n", "# `columns`: a list of column names for the returned DataFrame\n", "# `extension`: the file extension of the files to be read\n", "\n", - "# Using these principles, we can recreate a simple reader for reading subject weight data from the \n", + "# Using these principles, we can recreate a simple reader for reading subject weight data from the\n", "# social0.1 experiments, which are saved in .csv format.\n", "\n", + "\n", "# First, we'll create a general Csv reader, subclassed from `Reader`.\n", "class Csv(reader.Reader):\n", " \"\"\"Reads data from csv text files, where the first column stores the Aeon timestamp, in seconds.\"\"\"\n", @@ -161,10 +162,11 @@ "\n", " def read(self, file):\n", " return pd.read_csv(file, header=0, names=self.columns, index_col=0)\n", - " \n", + "\n", + "\n", "# Next, we'll create a reader for the subject weight data, subclassed from `Csv`.\n", "\n", - "# We know from our data that the files of interest start with 'Environment_SubjectWeight' and columns are: \n", + "# We know from our data that the files of interest start with 'Environment_SubjectWeight' and columns are:\n", "# 1) Aeon timestamp in seconds from 1904/01/01 (1904 date system)\n", "# 2) Weight in grams\n", "# 3) Weight stability confidence (0-1)\n", @@ -173,16 +175,17 @@ "# Since the first column (Aeon timestamp) will be set as the index, we'll use the rest as DataFrame columns.\n", "# And we don't need to define `read`, as it will use the `Csv` class's `read` method.\n", "\n", + "\n", "class Subject_Weight(Csv):\n", " \"\"\"Reads subject weight data from csv text files.\"\"\"\n", - " \n", + "\n", " def __init__(\n", - " self, \n", + " self,\n", " pattern=\"Environment_SubjectWeight*\",\n", - " columns=[\"weight\", \"confidence\", \"subject_id\", \"int_id\"], \n", - " extension=\"csv\"\n", + " columns=[\"weight\", \"confidence\", \"subject_id\", \"int_id\"],\n", + " extension=\"csv\",\n", " ):\n", - " super().__init__(pattern, columns, extension)\n" + " super().__init__(pattern, columns, extension)" ] }, { @@ -613,7 +616,7 @@ "source": [ "\"\"\"Loading data via a `Reader` object\"\"\"\n", "\n", - "# We can now load data by specifying a file \n", + "# We can now load data by specifying a file\n", "subject_weight_reader = Subject_Weight()\n", "acq_epoch = \"2023-12-01T14-30-34\"\n", "weight_file = root / acq_epoch / \"Environment/Environment_SubjectWeight_2023-12-02T12-00-00.csv\"\n", @@ -953,7 +956,7 @@ "source": [ "\"\"\"Updating a `Reader` object\"\"\"\n", "\n", - "# Occasionally, we may want to tweak the output from a `Reader` object's `read` method, or some tweaks to \n", + "# Occasionally, we may want to tweak the output from a `Reader` object's `read` method, or some tweaks to\n", "# streams on the acquisition side may require us to make corresponding tweaks to a `Reader` object to\n", "# ensure it works properly. We'll cover some of these cases here.\n", "\n", @@ -970,21 +973,23 @@ "\n", "# 2. Pattern changes\n", "\n", - "# Next, occasionally a stream's filename may change, in which case we'll need to update the `Reader` \n", - "# object's `pattern` to find the new files using `load`: \n", + "# Next, occasionally a stream's filename may change, in which case we'll need to update the `Reader`\n", + "# object's `pattern` to find the new files using `load`:\n", + "\n", "\n", "# Let's simulate a case where the old SubjectWeight stream was called Weight, and create a `Reader` class.\n", "class Subject_Weight(Csv):\n", " \"\"\"Reads subject weight data from csv text files.\"\"\"\n", - " \n", + "\n", " def __init__(\n", - " self, \n", + " self,\n", " pattern=\"Environment_Weight*\",\n", - " columns=[\"weight\", \"confidence\", \"subject_id\", \"int_id\"], \n", - " extension=\"csv\"\n", + " columns=[\"weight\", \"confidence\", \"subject_id\", \"int_id\"],\n", + " extension=\"csv\",\n", " ):\n", " super().__init__(pattern, columns, extension)\n", "\n", + "\n", "# We'll see that we can't find any files with this pattern.\n", "subject_weight_reader = Subject_Weight()\n", "data = aeon.load(root, subject_weight_reader, start=start_time, end=end_time)\n", @@ -993,7 +998,7 @@ "# But if we just update the pattern, `load` will find the files.\n", "subject_weight_reader.pattern = \"Environment_SubjectWeight*\"\n", "data = aeon.load(root, subject_weight_reader, start=start_time, end=end_time)\n", - "display(data) \n", + "display(data)\n", "\n", "\n", "# 3. Bitmask changes for Harp streams\n", @@ -1039,19 +1044,22 @@ "source": [ "\"\"\"Instantiating a `Device` object\"\"\"\n", "\n", - "# A `Device` object is instantiated from a name, followed by one or more 'binder functions', which \n", + "# A `Device` object is instantiated from a name, followed by one or more 'binder functions', which\n", "# return a dictionary of a name paired with a `Reader` object. We call such a dictionary of `:Reader`\n", "# key-value pairs a 'registry'. Each binder function requires a `pattern` argument, which can be used to\n", "# set the pattern of the `Reader` object it returns. This requirement for binder functions is for allowing\n", "# the `Device` to optionally pass its name to appropriately set the pattern of `Reader` objects it contains.\n", "\n", + "\n", "# Below are examples of \"empty pattern\" binder functions, where the pattern doesn't get used.\n", "def subject_weight_binder(pattern): # an example subject weight binder function\n", " return {\"subject_weight\": subject_weight_reader}\n", "\n", + "\n", "def subject_state_binder(pattern): # an example subject state binder function\n", " return {\"subject_state\": reader.Subject(pattern=\"Environment_SubjectState*\")}\n", "\n", + "\n", "d = Device(\"SubjectMetadata\", subject_weight_binder, subject_state_binder)\n", "\n", "# On creation, the `Device` object puts all registries into a single registry, which is accessible via the\n", @@ -1063,7 +1071,7 @@ "# for the `Device` object) are the `Reader` objects associated with that `Device` object.\n", "\n", "# This works because, when a list of `Device` objects are passed into the `DotMap` constructor, the\n", - "# `__iter__` method of the `Device` object returns a tuple of the object's name with its `stream` \n", + "# `__iter__` method of the `Device` object returns a tuple of the object's name with its `stream`\n", "# attribute, which is passed in directly to the DotMap constructor to create a nested DotMap:\n", "# device_name -> stream_name -> stream `Reader` object. This is shown below:\n", "\n", @@ -1095,7 +1103,8 @@ "# Binder functions can return a dict whose value is actually composed of multiple, rather than a single,\n", "# `Reader` objects. This is done by creating nested binder functions, via `register`.\n", "\n", - "# First let's define two standard binder functions, for pellet delivery trigger and beambreak events. \n", + "\n", + "# First let's define two standard binder functions, for pellet delivery trigger and beambreak events.\n", "# In all examples below we'll define \"device-name passed\" binder functions, since the `Device` object which\n", "# will be instantiated from these functions will pass its name to set the pattern of the corresponding\n", "# Reader objects.\n", @@ -1108,11 +1117,13 @@ " \"\"\"Pellet beambreak events.\"\"\"\n", " return {\"pellet_beambreak\": reader.BitmaskEvent(f\"{pattern}_32_*\", 0x22, \"PelletDetected\")}\n", "\n", + "\n", "# Next, we'll define a nested binder function for a \"feeder\", which returns the two binder functions above.\n", "def feeder(pattern):\n", " \"\"\"Feeder commands and events.\"\"\"\n", " return register(pattern, pellet_trigger, pellet_beambreak)\n", "\n", + "\n", "# And further, we can define a higher-level nested binder function for a \"patch\", which includes the\n", "# magnetic encoder values for a patch's wheel in addition to `feeder`.\n", "def patch(pattern):\n", @@ -1120,7 +1131,7 @@ " return register(pattern, feeder, core.encoder)\n", "\n", "\n", - "# We can now instantiate a `Device` object as done previously, from combinations of binder functions, but \n", + "# We can now instantiate a `Device` object as done previously, from combinations of binder functions, but\n", "# also from nested binder functions.\n", "feeder_device = Device(\"Patch1\", pellet_trigger, pellet_beambreak)\n", "feeder_device_nested = Device(\"Patch1\", feeder)\n", @@ -1272,20 +1283,10 @@ "subject_weight_b = lambda pattern: {\"SubjectWeight\": subject_weight_r} # binder function: \"empty pattern\"\n", "\n", "# Nested binder fn Device object.\n", - "environment = Device(\n", - " \"Environment\", # device name\n", - " env_block_state_b,\n", - " light_events_b,\n", - " core.message_log\n", - ")\n", + "environment = Device(\"Environment\", env_block_state_b, light_events_b, core.message_log) # device name\n", "\n", "# Separate Device object for subject-specific streams.\n", - "subject = Device(\n", - " \"Subject\",\n", - " subject_state_b,\n", - " subject_visits_b,\n", - " subject_weight_b\n", - ")\n", + "subject = Device(\"Subject\", subject_state_b, subject_visits_b, subject_weight_b)\n", "\n", "# ---\n", "\n", @@ -1299,12 +1300,7 @@ "cam_names = [\"Camera\" + name for name in cam_names]\n", "camera_b = [lambda pattern, name=name: {name: reader.Video(name + \"*\")} for name in cam_names]\n", "\n", - "camera = Device(\n", - " \"Camera\", \n", - " camera_top_b, \n", - " camera_top_pos_b, \n", - " *camera_b\n", - ")\n", + "camera = Device(\"Camera\", camera_top_b, camera_top_pos_b, *camera_b)\n", "\n", "# ---\n", "\n", @@ -1315,9 +1311,9 @@ "weight_filtered_b = lambda pattern: {\"WeightFiltered\": reader.Harp(\"Nest_202*\", [\"weight(g)\", \"stability\"])}\n", "\n", "nest = Device(\n", - " \"Nest\", \n", - " weight_raw_b, \n", - " weight_filtered_b, \n", + " \"Nest\",\n", + " weight_raw_b,\n", + " weight_filtered_b,\n", ")\n", "\n", "# ---\n", @@ -1352,10 +1348,7 @@ " }\n", " patch_b.append(fn)\n", "\n", - "patch = Device(\n", - " \"Patch\", \n", - " *patch_b\n", - ")\n", + "patch = Device(\"Patch\", *patch_b)\n", "# ---\n", "\n", "# Rfid\n", @@ -1365,10 +1358,7 @@ "rfid_names = [\"Rfid\" + name for name in rfid_names]\n", "rfid_b = [lambda pattern, name=name: {name: reader.Harp(name + \"*\", [\"rfid\"])} for name in rfid_names]\n", "\n", - "rfid = Device(\n", - " \"Rfid\", \n", - " *rfid_b\n", - ")\n", + "rfid = Device(\"Rfid\", *rfid_b)\n", "\n", "# ---" ] @@ -1379,17 +1369,7 @@ "metadata": {}, "outputs": [], "source": [ - "social01 = DotMap(\n", - " [\n", - " metadata,\n", - " environment,\n", - " subject,\n", - " camera,\n", - " nest,\n", - " patch,\n", - " rfid\n", - " ]\n", - ")" + "social01 = DotMap([metadata, environment, subject, camera, nest, patch, rfid])" ] }, { @@ -4125,6 +4105,7 @@ "source": [ "\"\"\"Test all readers in schema.\"\"\"\n", "\n", + "\n", "def find_obj(dotmap, obj):\n", " \"\"\"Returns a list of objects of type `obj` found in a DotMap.\"\"\"\n", " objs = []\n", @@ -4135,12 +4116,13 @@ " objs.extend(find_obj(value, obj))\n", " return objs\n", "\n", + "\n", "readers = find_obj(social01, reader.Reader)\n", "start_time = pd.Timestamp(\"2023-12-05 15:00:00\")\n", "end_time = pd.Timestamp(\"2023-12-07 11:00:00\")\n", "for r in readers:\n", " data = aeon.load(root, r, start=start_time, end=end_time)\n", - " #assert not data.empty, f\"No data found with {r}.\"\n", + " # assert not data.empty, f\"No data found with {r}.\"\n", " print(f\"\\n{r.pattern}:\")\n", " display(data.head())" ] diff --git a/env_config/env.yml b/env_config/env.yml deleted file mode 100644 index 9a0aeb32..00000000 --- a/env_config/env.yml +++ /dev/null @@ -1,37 +0,0 @@ -# Create env e.g. w/mamba: `mamba env create -q -f env.yml` -# Update exisiting env e.g. w/mamba: `mamba env update -f env.yml` - -name: aeon -channels: - - conda-forge - - defaults -dependencies: - - python>=3.11 - - pip - - blas>=2.0, <3 - - bottleneck>=1.2.1, <2 - - dash - - dotmap - - fastparquet - - graphviz - - ipykernel - - jupyter - - jupyterlab - - matplotlib - - numba>=0.46.0, <1 - - numexpr>=2.6.8, <3 - - numpy>=1.21.0, <2 - - pandas>=1.3 - - plotly - - pyarrow - - pydotplus - - pymysql - - pyyaml - - scikit-learn - - scipy - - seaborn - - xarray>=0.12.3, <1 - - pip: - - datajoint>=0.13.6, <1 - - git+https://github.com/datajoint-company/datajoint-utilities.git - - opencv-python diff --git a/env_config/env_dev.yml b/env_config/env_dev.yml deleted file mode 100644 index 19aeeab2..00000000 --- a/env_config/env_dev.yml +++ /dev/null @@ -1,24 +0,0 @@ -# Contains only the dev package requirements. -# Create env e.g. w/mamba: `mamba env create -q -f env.yml` -# Update exisiting env e.g. w/mamba: `mamba env update -f env_dev.yml` - -name: aeon -channels: - - conda-forge - - defaults -dependencies: - - black[jupyter] - - gh - - ipdb - - jellyfish - - pre-commit - - pydantic - - pyright - - pytest - - pytest-cov - - ruff - - sphinx - - tox - - pip: - - git+https://github.com/Technologicat/pyan.git - diff --git a/env_config/env_gpu.yml b/env_config/env_gpu.yml deleted file mode 100644 index ef045e76..00000000 --- a/env_config/env_gpu.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Contains only the gpu package requirements. -# Create env e.g. w/mamba: `mamba env create -q -f env.yml` -# Update exisiting env e.g. w/mamba: `mamba env update -f env_gpu.yml` - -name: aeon -channels: - - conda-forge - - defaults -dependencies: - - cupy - - dask diff --git a/pyproject.toml b/pyproject.toml index 720ee826..658abf5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,8 +22,8 @@ license = { file = "license.md" } readme = "readme.md" dependencies = [ "bottleneck>=1.2.1,<2", - "datajoint-utilities @ git+https://github.com/datajoint-company/datajoint-utilities", "datajoint>=0.13.7", + "datajoint-utilities @ git+https://github.com/datajoint-company/datajoint-utilities", "dotmap", "fastparquet", "graphviz", @@ -51,7 +51,6 @@ dependencies = [ [project.optional-dependencies] dev = [ "bandit", - "black[jupyter]", "gh", "ipdb", "pre-commit", @@ -78,26 +77,8 @@ DataJoint = "https://docs.datajoint.org/" [tool.setuptools.packages.find] include = ["aeon*"] -[tool.black] -line-length = 108 -color = false -exclude = ''' -/( - \.git - | \.mypy_cache - | \.tox - | \.venv - | _build - | build - | dist - | env - | venv -)/ -''' -extend-exclude = "aeon/dj_pipeline/streams.py" - [tool.ruff] -select = [ +lint.select = [ "E", "W", "F", @@ -115,7 +96,11 @@ select = [ "PL", ] line-length = 108 -ignore = [ +lint.ignore = [ + "D100", # skip adding docstrings for module + "D104", # ignore missing docstring in public package + "D105", # skip adding docstrings for magic methods + "D107", # skip adding docstrings for __init__ "E201", "E202", "E203", @@ -123,6 +108,7 @@ ignore = [ "E731", "E702", "S101", + "PT004", # Rule `PT004` is deprecated and will be removed in a future release. "PT013", "PLR0912", "PLR0913", @@ -132,18 +118,24 @@ extend-exclude = [ ".git", ".github", ".idea", + "*.ipynb", ".vscode", "aeon/dj_pipeline/streams.py", ] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] +"tests/*" = [ + "D103", # skip adding docstrings for public functions +] +"aeon/schema/*" = [ + "D101", # skip adding docstrings for schema classes + "D106", # skip adding docstrings for nested streams +] "aeon/dj_pipeline/*" = [ "B006", "B021", - "D100", # skip adding docstrings for module "D101", # skip adding docstrings for table class since it is added inside definition "D102", # skip adding docstrings for make function "D103", # skip adding docstrings for public functions - "D104", # ignore missing docstring in public package "D106", # skip adding docstrings for Part tables "E501", "F401", # ignore unused import errors @@ -167,10 +159,11 @@ extend-exclude = [ "I001", ] -[tool.ruff.pydocstyle] +[tool.ruff.lint.pydocstyle] convention = "google" [tool.pyright] +useLibraryCodeForTypes = false reportMissingImports = "none" reportImportCycles = "error" reportUnusedImport = "error" @@ -193,6 +186,6 @@ reportShadowedImports = "error" # *Note*: we may want to set all 'ReportOptional*' rules to "none", but leaving 'em default for now venvPath = "." venv = ".venv" -exclude = ["aeon/dj_pipeline/*"] +exclude = ["aeon/dj_pipeline/*", ".venv/*"] [tool.pytest.ini_options] markers = ["api"] diff --git a/tests/conftest.py b/tests/dj_pipeline/conftest.py similarity index 92% rename from tests/conftest.py rename to tests/dj_pipeline/conftest.py index 4236c3e3..3604890f 100644 --- a/tests/conftest.py +++ b/tests/dj_pipeline/conftest.py @@ -1,4 +1,5 @@ -""" +"""Global configurations and fixtures for pytest. + # run all tests: # pytest -sv --cov-report term-missing --cov=aeon_mecha -p no:warnings tests/dj_pipeline @@ -20,15 +21,12 @@ def data_dir(): - """ - Returns test data directory - """ + """Returns test data directory.""" return os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") @pytest.fixture(autouse=True, scope="session") def test_params(): - return { "start_ts": "2022-06-22 08:51:10", "end_ts": "2022-06-22 14:00:00", @@ -49,28 +47,17 @@ def test_params(): @pytest.fixture(autouse=True, scope="session") def dj_config(): - """If dj_local_config exists, load""" + """Configures DataJoint connection and loads custom settings.""" dj_config_fp = pathlib.Path("dj_local_conf.json") assert dj_config_fp.exists() dj.config.load(dj_config_fp) dj.config["safemode"] = False assert "custom" in dj.config - dj.config["custom"][ - "database.prefix" - ] = f"u_{dj.config['database.user']}_testsuite_" + dj.config["custom"]["database.prefix"] = f"u_{dj.config['database.user']}_testsuite_" def load_pipeline(): - - from aeon.dj_pipeline import ( - acquisition, - analysis, - lab, - qc, - report, - subject, - tracking, - ) + from aeon.dj_pipeline import acquisition, analysis, lab, qc, report, subject, tracking return { "subject": subject, @@ -84,7 +71,6 @@ def load_pipeline(): def drop_schema(): - _pipeline = load_pipeline() _pipeline["report"].schema.drop() @@ -100,7 +86,6 @@ def drop_schema(): @pytest.fixture(autouse=True, scope="session") def pipeline(dj_config): - _pipeline = load_pipeline() yield _pipeline diff --git a/tests/dj_pipeline/test_acquisition.py b/tests/dj_pipeline/test_acquisition.py index 8e5eec37..28f39d12 100644 --- a/tests/dj_pipeline/test_acquisition.py +++ b/tests/dj_pipeline/test_acquisition.py @@ -16,29 +16,18 @@ def test_epoch_chunk_ingestion(test_params, pipeline, epoch_chunk_ingestion): @mark.ingestion -def test_experimentlog_ingestion( - test_params, pipeline, epoch_chunk_ingestion, experimentlog_ingestion -): +def test_experimentlog_ingestion(test_params, pipeline, epoch_chunk_ingestion, experimentlog_ingestion): acquisition = pipeline["acquisition"] assert ( - len( - acquisition.ExperimentLog.Message - & {"experiment_name": test_params["experiment_name"]} - ) + len(acquisition.ExperimentLog.Message & {"experiment_name": test_params["experiment_name"]}) == test_params["experiment_log_message_count"] ) assert ( - len( - acquisition.SubjectEnterExit.Time - & {"experiment_name": test_params["experiment_name"]} - ) + len(acquisition.SubjectEnterExit.Time & {"experiment_name": test_params["experiment_name"]}) == test_params["subject_enter_exit_count"] ) assert ( - len( - acquisition.SubjectWeight.WeightTime - & {"experiment_name": test_params["experiment_name"]} - ) + len(acquisition.SubjectWeight.WeightTime & {"experiment_name": test_params["experiment_name"]}) == test_params["subject_weight_time_count"] ) diff --git a/tests/dj_pipeline/test_pipeline_instantiation.py b/tests/dj_pipeline/test_pipeline_instantiation.py index 30377d95..cb3b51fb 100644 --- a/tests/dj_pipeline/test_pipeline_instantiation.py +++ b/tests/dj_pipeline/test_pipeline_instantiation.py @@ -3,28 +3,24 @@ @mark.instantiation def test_pipeline_instantiation(pipeline): - assert hasattr(pipeline["acquisition"], "FoodPatchEvent") assert hasattr(pipeline["lab"], "Arena") assert hasattr(pipeline["qc"], "CameraQC") assert hasattr(pipeline["report"], "InArenaSummaryPlot") assert hasattr(pipeline["subject"], "Subject") assert hasattr(pipeline["tracking"], "CameraTracking") - - + + @mark.instantiation def test_experiment_creation(test_params, pipeline, experiment_creation): acquisition = pipeline["acquisition"] - + experiment_name = test_params["experiment_name"] assert acquisition.Experiment.fetch1("experiment_name") == experiment_name raw_dir = ( - acquisition.Experiment.Directory - & {"experiment_name": experiment_name, "directory_type": "raw"} + acquisition.Experiment.Directory & {"experiment_name": experiment_name, "directory_type": "raw"} ).fetch1("directory_path") assert raw_dir == test_params["raw_dir"] - exp_subjects = ( - acquisition.Experiment.Subject & {"experiment_name": experiment_name} - ).fetch("subject") + exp_subjects = (acquisition.Experiment.Subject & {"experiment_name": experiment_name}).fetch("subject") assert len(exp_subjects) == test_params["subject_count"] assert "BAA-1100701" in exp_subjects diff --git a/tests/dj_pipeline/test_qc.py b/tests/dj_pipeline/test_qc.py index bfe248fc..9815031e 100644 --- a/tests/dj_pipeline/test_qc.py +++ b/tests/dj_pipeline/test_qc.py @@ -3,7 +3,6 @@ @mark.qc def test_camera_qc_ingestion(test_params, pipeline, camera_qc_ingestion): - qc = pipeline["qc"] assert len(qc.CameraQC()) == test_params["camera_qc_count"] diff --git a/tests/dj_pipeline/test_tracking.py b/tests/dj_pipeline/test_tracking.py index 5920bfd8..973e0741 100644 --- a/tests/dj_pipeline/test_tracking.py +++ b/tests/dj_pipeline/test_tracking.py @@ -6,21 +6,20 @@ index = 0 column_name = "position_x" # data column to run test on -file_name = "exp0.2-r0-20220524090000-21053810-20220524082942-0-0.npy" # test file to be saved with save_test_data +file_name = ( + "exp0.2-r0-20220524090000-21053810-20220524082942-0-0.npy" # test file to be saved with save_test_data +) def save_test_data(pipeline, test_params): - """save test dataset fetched from tracking.CameraTracking.Object""" - + """Save test dataset fetched from tracking.CameraTracking.Object.""" tracking = pipeline["tracking"] key = tracking.CameraTracking.Object().fetch("KEY")[index] file_name = ( "-".join( [ - v.strftime("%Y%m%d%H%M%S") - if isinstance(v, datetime.datetime) - else str(v) + v.strftime("%Y%m%d%H%M%S") if isinstance(v, datetime.datetime) else str(v) for v in key.values() ] ) @@ -37,21 +36,15 @@ def save_test_data(pipeline, test_params): @mark.ingestion @mark.tracking def test_camera_tracking_ingestion(test_params, pipeline, camera_tracking_ingestion): - tracking = pipeline["tracking"] - assert ( - len(tracking.CameraTracking.Object()) - == test_params["camera_tracking_object_count"] - ) + assert len(tracking.CameraTracking.Object()) == test_params["camera_tracking_object_count"] key = tracking.CameraTracking.Object().fetch("KEY")[index] file_name = ( "-".join( [ - v.strftime("%Y%m%d%H%M%S") - if isinstance(v, datetime.datetime) - else str(v) + v.strftime("%Y%m%d%H%M%S") if isinstance(v, datetime.datetime) else str(v) for v in key.values() ] ) diff --git a/tests/io/test_api.py b/tests/io/test_api.py index b3cb3302..095439de 100644 --- a/tests/io/test_api.py +++ b/tests/io/test_api.py @@ -36,7 +36,8 @@ def test_load_filter_nonchunked(): @mark.api def test_load_monotonic(): data = aeon.load(monotonic_path, exp02.Patch2.Encoder, downsample=None) - assert len(data) > 0 and data.index.is_monotonic_increasing + assert len(data) > 0 + assert data.index.is_monotonic_increasing @mark.api @@ -47,6 +48,7 @@ def test_load_nonmonotonic(): @mark.api def test_load_encoder_with_downsampling(): + DOWNSAMPLE_PERIOD = 0.02 data = aeon.load(monotonic_path, exp02.Patch2.Encoder, downsample=True) raw_data = aeon.load(monotonic_path, exp02.Patch2.Encoder, downsample=None) @@ -54,14 +56,14 @@ def test_load_encoder_with_downsampling(): assert len(data) < len(raw_data) # Check that the first timestamp of the downsampled data is within 20ms of the raw data - assert abs(data.index[0] - raw_data.index[0]).total_seconds() <= 0.02 + assert abs(data.index[0] - raw_data.index[0]).total_seconds() <= DOWNSAMPLE_PERIOD # Check that the last timestamp of the downsampled data is within 20ms of the raw data - assert abs(data.index[-1] - raw_data.index[-1]).total_seconds() <= 0.02 + assert abs(data.index[-1] - raw_data.index[-1]).total_seconds() <= DOWNSAMPLE_PERIOD # Check that the minimum difference between consecutive timestamps in the downsampled data # is at least 20ms (50Hz) - assert data.index.to_series().diff().dt.total_seconds().min() >= 0.02 + assert data.index.to_series().diff().dt.total_seconds().min() >= DOWNSAMPLE_PERIOD # Check that the timestamps in the downsampled data are strictly increasing assert data.index.is_monotonic_increasing