From f3612bd668f622c45f318d1c493011752a24b001 Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Sat, 31 Aug 2024 11:15:58 -0400 Subject: [PATCH] addd 000458/FlatironInstitute --- .../000_lindi_vs_fsspec_streaming.ipynb | 146 +++++ .../001_summarize_contents.ipynb | 578 ++++++++++++++++++ .../PSTH_neurosift_exploration.md | 39 ++ 000458/FlatironInstitute/README.md | 7 + 000458/FlatironInstitute/environment.yaml | 16 + .../helpers/stream_nwbfile.py | 35 ++ 6 files changed, 821 insertions(+) create mode 100644 000458/FlatironInstitute/000_lindi_vs_fsspec_streaming.ipynb create mode 100644 000458/FlatironInstitute/001_summarize_contents.ipynb create mode 100644 000458/FlatironInstitute/PSTH_neurosift_exploration.md create mode 100644 000458/FlatironInstitute/README.md create mode 100644 000458/FlatironInstitute/environment.yaml create mode 100644 000458/FlatironInstitute/helpers/stream_nwbfile.py diff --git a/000458/FlatironInstitute/000_lindi_vs_fsspec_streaming.ipynb b/000458/FlatironInstitute/000_lindi_vs_fsspec_streaming.ipynb new file mode 100644 index 0000000..db86115 --- /dev/null +++ b/000458/FlatironInstitute/000_lindi_vs_fsspec_streaming.ipynb @@ -0,0 +1,146 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/magland/miniconda3/envs/000458_fi/lib/python3.12/site-packages/hdmf/utils.py:668: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.5.1 because version 1.8.0 is already loaded.\n", + " return func(args[0], **pargs)\n", + "/home/magland/miniconda3/envs/000458_fi/lib/python3.12/site-packages/hdmf/utils.py:668: UserWarning: Ignoring cached namespace 'core' version 2.5.0 because version 2.7.0 is already loaded.\n", + " return func(args[0], **pargs)\n", + "/home/magland/miniconda3/envs/000458_fi/lib/python3.12/site-packages/hdmf/utils.py:668: UserWarning: Ignoring cached namespace 'hdmf-experimental' version 0.2.0 because version 0.5.0 is already loaded.\n", + " return func(args[0], **pargs)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Elapsed time for lindi: 7.52 s\n", + "Elapsed time for fsspec: 141.01 s\n" + ] + } + ], + "source": [ + "# This notebook compares the time taken to stream an NWB file from DANDI archive\n", + "# using lindi and fsspec. On my laptop on my home WiFi network, lindi streaming\n", + "# took 7.52 s and fsspec streaming took 141.01 s. This is expected to depend\n", + "# heavily on the network properties.\n", + "\n", + "import time\n", + "from pynwb import NWBHDF5IO\n", + "from dandi.dandiapi import DandiAPIClient\n", + "from fsspec import filesystem\n", + "from h5py import File\n", + "import lindi\n", + "\n", + "\n", + "def stream_nwbfile(DANDISET_ID, file_path):\n", + " '''Stream NWB file from DANDI archive.\n", + "\n", + " Parameters\n", + " ----------\n", + " DANDISET_ID : str\n", + " Dandiset ID\n", + " file_path : str\n", + " Path to NWB file in DANDI archive\n", + "\n", + " Returns\n", + " -------\n", + " nwbfile : NWBFile\n", + " NWB file\n", + " io : NWBHDF5IO\n", + " NWB IO object (for closing)\n", + "\n", + " Notes\n", + " -----\n", + " The io object must be closed after use.\n", + " '''\n", + " with DandiAPIClient() as client:\n", + " asset = client.get_dandiset(DANDISET_ID, 'draft').get_asset_by_path(file_path)\n", + " asset_url = asset.get_content_url(follow_redirects=0, strip_query=True)\n", + " file = lindi.LindiH5pyFile.from_hdf5_file(asset_url)\n", + " io = NWBHDF5IO(file=file, load_namespaces=True)\n", + " nwbfile = io.read()\n", + " return nwbfile, io\n", + "\n", + "\n", + "def stream_nwbfile_fsspec(DANDISET_ID, file_path):\n", + " '''Stream NWB file from DANDI archive.\n", + "\n", + " Parameters\n", + " ----------\n", + " DANDISET_ID : str\n", + " Dandiset ID\n", + " file_path : str\n", + " Path to NWB file in DANDI archive\n", + "\n", + " Returns\n", + " -------\n", + " nwbfile : NWBFile\n", + " NWB file\n", + " io : NWBHDF5IO\n", + " NWB IO object (for closing)\n", + "\n", + " Notes\n", + " -----\n", + " The io object must be closed after use.\n", + " '''\n", + " with DandiAPIClient() as client:\n", + " asset = client.get_dandiset(DANDISET_ID, 'draft').get_asset_by_path(file_path)\n", + " asset_url = asset.get_content_url(follow_redirects=0, strip_query=True)\n", + " fs = filesystem(\"http\")\n", + " file_system = fs.open(asset_url, \"rb\")\n", + " file = File(file_system, mode=\"r\")\n", + " io = NWBHDF5IO(file=file, load_namespaces=True)\n", + " nwbfile = io.read()\n", + " return nwbfile, io\n", + "\n", + "\n", + "DANDISET_ID = \"000458\"\n", + "file_path = \"sub-586468/sub-586468_ses-20210819_behavior+ecephys.nwb\"\n", + "\n", + "timer = time.time()\n", + "nwbfile, io = stream_nwbfile(DANDISET_ID, file_path)\n", + "print(f\"Elapsed time for lindi: {time.time() - timer:.2f} s\")\n", + "\n", + "timer = time.time()\n", + "nwbfile, io = stream_nwbfile_fsspec(DANDISET_ID, file_path)\n", + "print(f\"Elapsed time for fsspec: {time.time() - timer:.2f} s\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "000458_fi", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/000458/FlatironInstitute/001_summarize_contents.ipynb b/000458/FlatironInstitute/001_summarize_contents.ipynb new file mode 100644 index 0000000..e77d1f9 --- /dev/null +++ b/000458/FlatironInstitute/001_summarize_contents.ipynb @@ -0,0 +1,578 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 000458 - summarize contents\n", + "\n", + "This notebook summarizes the contents of all the NWB files in Dandiset 000458." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Supress hdmf warnings about cached namespaces\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\", category=UserWarning, module=\"hdmf.utils\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Asset 1: sub-521885/sub-521885_ses-20200709_behavior+ecephys.nwb -- size: 312.14 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (superficial layers)\n", + " 360 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 154 valid and not running\n", + " isoflurane: 174 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 328 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 328 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 328 valid and not running\n", + "\n", + "\n", + "Asset 2: sub-521886/sub-521886_ses-20200716_behavior+ecephys.nwb -- size: 504.31 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (superficial layers)\n", + " 900 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 262 valid and not running\n", + " isoflurane: 300 valid and not running\n", + " recovery: 264 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 826 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 826 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 826 valid and not running\n", + "\n", + "\n", + "Asset 3: sub-521887/sub-521887_ses-20200730_behavior+ecephys.nwb -- size: 428.19 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (superficial layers)\n", + " 800 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 148 valid and not running\n", + " isoflurane: 300 valid and not running\n", + " recovery: 278 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 726 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 726 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 726 valid and not running\n", + "\n", + "\n", + "Asset 4: sub-543393/sub-543393_ses-20200820_behavior+ecephys.nwb -- size: 527.51 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (superficial layers)\n", + " 900 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 246 valid and not running\n", + " isoflurane: 300 valid and not running\n", + " recovery: 276 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 822 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 822 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 822 valid and not running\n", + "\n", + "\n", + "Asset 5: sub-543394/sub-543394_ses-20200827_behavior+ecephys.nwb -- size: 589.19 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (superficial layers)\n", + " 900 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 192 valid and not running\n", + " isoflurane: 300 valid and not running\n", + " recovery: 281 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 773 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 773 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 773 valid and not running\n", + "\n", + "\n", + "Asset 6: sub-546655/sub-546655_ses-20201023_behavior+ecephys.nwb -- size: 14872.63 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (deep layers)\n", + " 1459 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 121 valid and not running\n", + " isoflurane: 397 valid and not running\n", + " recovery: 258 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 641 valid and not running\n", + " visual: 135 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 641 valid and not running\n", + " white fullscreen: 135 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 641 valid and not running\n", + " n/a: 135 valid and not running\n", + "\n", + "\n", + "Asset 7: sub-551397/sub-551397_ses-20210211_behavior+ecephys.nwb -- size: 15202.44 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (deep layers)\n", + " 1800 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 386 valid and not running\n", + " isoflurane: 480 valid and not running\n", + " recovery: 553 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 1103 valid and not running\n", + " visual: 316 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 1103 valid and not running\n", + " white circle: 316 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 1103 valid and not running\n", + " n/a: 316 valid and not running\n", + "\n", + "\n", + "Asset 8: sub-551399/sub-551399_ses-20210128_behavior+ecephys.nwb -- size: 11687.84 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (deep layers)\n", + " 1440 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 323 valid and not running\n", + " isoflurane: 359 valid and not running\n", + " recovery: 654 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 1336 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 1336 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 1336 valid and not running\n", + "\n", + "\n", + "Asset 9: sub-569062/sub-569062_ses-20210218_behavior+ecephys.nwb -- size: 24829.25 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (deep layers)\n", + " 1440 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 398 valid and not running\n", + " isoflurane: 479 valid and not running\n", + " recovery: 449 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 985 valid and not running\n", + " visual: 341 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 985 valid and not running\n", + " white circle: 341 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 985 valid and not running\n", + " n/a: 341 valid and not running\n", + "\n", + "\n", + "Asset 10: sub-569064/sub-569064_ses-20210408_behavior+ecephys.nwb -- size: 25890.18 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (deep layers)\n", + " 1920 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 468 valid and not running\n", + " isoflurane: 477 valid and not running\n", + " recovery: 877 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 1350 valid and not running\n", + " visual: 472 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 1350 valid and not running\n", + " white circle: 472 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 1350 valid and not running\n", + " n/a: 472 valid and not running\n", + "\n", + "\n", + "Asset 11: sub-569068/sub-569068_ses-20210304_behavior+ecephys.nwb -- size: 27288.59 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (deep layers)\n", + " 1800 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 380 valid and not running\n", + " isoflurane: 477 valid and not running\n", + " recovery: 516 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 1119 valid and not running\n", + " visual: 254 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 1119 valid and not running\n", + " white circle: 254 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 1119 valid and not running\n", + " n/a: 254 valid and not running\n", + "\n", + "\n", + "Asset 12: sub-569069/sub-569069_ses-20210312_behavior+ecephys.nwb -- size: 22471.66 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (deep layers)\n", + " 1800 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 309 valid and not running\n", + " isoflurane: 471 valid and not running\n", + " recovery: 658 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 1173 valid and not running\n", + " visual: 265 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 1173 valid and not running\n", + " white circle: 265 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 1173 valid and not running\n", + " n/a: 265 valid and not running\n", + "\n", + "\n", + "Asset 13: sub-569070/sub-569070_ses-20210401_behavior+ecephys.nwb -- size: 1124.31 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (deep layers)\n", + " 1440 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 273 valid and not running\n", + " isoflurane: 360 valid and not running\n", + " recovery: 0 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 633 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 633 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 633 valid and not running\n", + "\n", + "\n", + "Asset 14: sub-569072/sub-569072_ses-20210422_behavior+ecephys.nwb -- size: 13304.27 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to SSp (deep layers)\n", + " 960 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 430 valid and not running\n", + " isoflurane: 477 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 680 valid and not running\n", + " visual: 227 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 680 valid and not running\n", + " white circle: 227 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " SSp: 680 valid and not running\n", + " n/a: 227 valid and not running\n", + "\n", + "\n", + "Asset 15: sub-569073/sub-569073_ses-20210415_behavior+ecephys.nwb -- size: 26729.21 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (deep layers)\n", + " 2040 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 383 valid and not running\n", + " isoflurane: 600 valid and not running\n", + " recovery: 810 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 1279 valid and not running\n", + " visual: 514 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 1279 valid and not running\n", + " white circle: 514 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 1279 valid and not running\n", + " n/a: 514 valid and not running\n", + "\n", + "\n", + "Asset 16: sub-569073/sub-569073_ses-20210416_behavior+ecephys.nwb -- size: 13200.52 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to SSp (deep layers)\n", + " 720 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 269 valid and not running\n", + " isoflurane: 360 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 629 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 629 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " SSp: 629 valid and not running\n", + "\n", + "\n", + "Asset 17: sub-571619/sub-571619_ses-20210319_behavior+ecephys.nwb -- size: 23171.96 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (superficial and deep layers)\n", + " 1440 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 577 valid and not running\n", + " isoflurane: 719 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 1296 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 1296 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 1296 valid and not running\n", + "\n", + "\n", + "Asset 18: sub-571620/sub-571620_ses-20210513_behavior+ecephys.nwb -- size: 23246.55 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to SSp (superficial and deep layers)\n", + " 1680 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 811 valid and not running\n", + " isoflurane: 720 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 1306 valid and not running\n", + " visual: 225 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 1306 valid and not running\n", + " white circle: 225 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " SSp: 1306 valid and not running\n", + " n/a: 225 valid and not running\n", + "\n", + "\n", + "Asset 19: sub-575102/sub-575102_ses-20210603_behavior+ecephys.nwb -- size: 24332.63 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to MOs (superficial layers)\n", + " 1882 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 354 valid and not running\n", + " isoflurane: 480 valid and not running\n", + " recovery: 223 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 826 valid and not running\n", + " visual: 231 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 826 valid and not running\n", + " white circle: 231 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " MOs: 826 valid and not running\n", + " n/a: 231 valid and not running\n", + "\n", + "\n", + "Asset 20: sub-586466/sub-586466_ses-20210729_behavior+ecephys.nwb -- size: 14282.62 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to SSp (superficial and deep layers)\n", + " 1680 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 867 valid and not running\n", + " isoflurane: 712 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 1377 valid and not running\n", + " visual: 202 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 1377 valid and not running\n", + " white circle: 202 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " SSp: 1377 valid and not running\n", + " n/a: 202 valid and not running\n", + "\n", + "\n", + "Asset 21: sub-586468/sub-586468_ses-20210819_behavior+ecephys.nwb -- size: 22105.06 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to SSp (superficial and deep layers)\n", + " 1680 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 410 valid and not running\n", + " isoflurane: 360 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 650 valid and not running\n", + " visual: 120 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 650 valid and not running\n", + " white circle: 120 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " SSp: 650 valid and not running\n", + " n/a: 120 valid and not running\n", + "\n", + "\n", + "Asset 22: sub-590479/sub-590479_ses-20210921_behavior+ecephys.nwb -- size: 12027.34 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to SSp (superficial and deep layers)\n", + " 1440 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 593 valid and not running\n", + " isoflurane: 719 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 1312 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 1312 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " SSp: 1312 valid and not running\n", + "\n", + "\n", + "Asset 23: sub-590480/sub-590480_ses-20211111_behavior+ecephys.nwb -- size: 21088.74 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to SSp (superficial and deep layers)\n", + " 1680 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 473 valid and not running\n", + " isoflurane: 720 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 1028 valid and not running\n", + " visual: 165 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 1028 valid and not running\n", + " white circle: 165 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " SSp: 1028 valid and not running\n", + " n/a: 165 valid and not running\n", + "\n", + "\n", + "Asset 24: sub-599017/sub-599017_ses-20220210_behavior+ecephys.nwb -- size: 22265.17 MB\n", + "in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation\n", + "EEG and Neuropixels recording during wakefulness and isoflurane anesthesia\n", + "single pulse electrical stimuli targeted to SSp (superficial and deep layers)\n", + " 1680 trials\n", + " BEHAVIORAL EPOCHS\n", + " awake: 254 valid and not running\n", + " isoflurane: 720 valid and not running\n", + " STIMULUS TYPES\n", + " electrical: 828 valid and not running\n", + " visual: 146 valid and not running\n", + " STIMULUS DESCRIPTIONS\n", + " biphasic: 828 valid and not running\n", + " white circle: 146 valid and not running\n", + " ESTIM TARGET REGIONS\n", + " SSp: 828 valid and not running\n", + " n/a: 146 valid and not running\n", + "\n" + ] + } + ], + "source": [ + "from helpers.stream_nwbfile import stream_nwbfile\n", + "import dandi.dandiarchive as da\n", + "\n", + "\n", + "DANDISET_ID = \"000458\"\n", + "\n", + "parsed_url = da.parse_dandi_url(f\"https://dandiarchive.org/dandiset/{DANDISET_ID}\")\n", + "\n", + "with parsed_url.navigate() as (client, dandiset, assets):\n", + " if dandiset is None:\n", + " raise Exception(f\"Dandiset {DANDISET_ID} not found.\")\n", + "\n", + " num_consecutive_not_nwb = 0\n", + " asset_index = 0\n", + " for asset_obj in dandiset.get_assets('path'):\n", + " if not asset_obj.path.endswith(\".nwb\"):\n", + " num_consecutive_not_nwb += 1\n", + " if num_consecutive_not_nwb >= 20:\n", + " # For example, this is important for 000026 because there are so many non-nwb assets\n", + " raise Exception(\"Too many consecutive non-NWB files.\")\n", + " continue\n", + " else:\n", + " num_consecutive_not_nwb = 0\n", + " asset = {\n", + " \"identifier\": asset_obj.identifier,\n", + " \"path\": asset_obj.path,\n", + " \"size\": asset_obj.size,\n", + " \"download_url\": asset_obj.download_url,\n", + " \"dandiset_id\": DANDISET_ID,\n", + " }\n", + " nwbfile, io = stream_nwbfile(DANDISET_ID, asset[\"path\"])\n", + " print('')\n", + " print(f'Asset {asset_index + 1}: {asset[\"path\"]} -- size: {asset[\"size\"] / 1e6:.2f} MB')\n", + " experiment_description = nwbfile.experiment_description # type: ignore\n", + " session_description = nwbfile.session_description # type: ignore\n", + " stimulus_notes = nwbfile.stimulus_notes # type: ignore\n", + " print(experiment_description)\n", + " print(session_description)\n", + " print(stimulus_notes)\n", + " trials = nwbfile.intervals['trials'].to_dataframe() # type: ignore\n", + " num_trials = len(trials)\n", + " print(f\" {num_trials} trials\")\n", + " behavioral_epochs = trials['behavioral_epoch']\n", + " is_valid = trials['is_valid']\n", + " is_running = trials['is_running']\n", + " unique_behavioral_epochs = behavioral_epochs.unique()\n", + " print(' BEHAVIORAL EPOCHS')\n", + " for be in unique_behavioral_epochs:\n", + " num = len(behavioral_epochs[behavioral_epochs == be])\n", + " num_valid = len(is_valid[(behavioral_epochs == be) & is_valid])\n", + " num_valid_and_not_running = len(is_valid[(behavioral_epochs == be) & is_valid & ~is_running])\n", + " print(f\" {be}: {num_valid_and_not_running} valid and not running\")\n", + " stimulus_types = trials['stimulus_type']\n", + " unique_stimulus_types = stimulus_types.unique()\n", + " print(' STIMULUS TYPES')\n", + " for st in unique_stimulus_types:\n", + " num = len(stimulus_types[stimulus_types == st])\n", + " num_valid = len(is_valid[(stimulus_types == st) & is_valid])\n", + " num_valid_and_not_running = len(is_valid[(stimulus_types == st) & is_valid & ~is_running])\n", + " print(f\" {st}: {num_valid_and_not_running} valid and not running\")\n", + " stimulus_descriptions = trials['stimulus_description']\n", + " unique_stimulus_descriptions = stimulus_descriptions.unique()\n", + " print(' STIMULUS DESCRIPTIONS')\n", + " for sd in unique_stimulus_descriptions:\n", + " num = len(stimulus_descriptions[stimulus_descriptions == sd])\n", + " num_valid = len(is_valid[(stimulus_descriptions == sd) & is_valid])\n", + " num_valid_and_not_running = len(is_valid[(stimulus_descriptions == sd) & is_valid & ~is_running])\n", + " print(f\" {sd}: {num_valid_and_not_running} valid and not running\")\n", + " estim_target_regions = trials['estim_target_region']\n", + " unique_estim_target_regions = estim_target_regions.unique()\n", + " print(' ESTIM TARGET REGIONS')\n", + " for etr in unique_estim_target_regions:\n", + " num = len(estim_target_regions[estim_target_regions == etr])\n", + " num_valid = len(is_valid[(estim_target_regions == etr) & is_valid])\n", + " num_valid_and_not_running = len(is_valid[(estim_target_regions == etr) & is_valid & ~is_running])\n", + " print(f\" {etr}: {num_valid_and_not_running} valid and not running\")\n", + " print('')\n", + " asset_index += 1" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "000458_fi", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/000458/FlatironInstitute/PSTH_neurosift_exploration.md b/000458/FlatironInstitute/PSTH_neurosift_exploration.md new file mode 100644 index 0000000..0ba71cb --- /dev/null +++ b/000458/FlatironInstitute/PSTH_neurosift_exploration.md @@ -0,0 +1,39 @@ +# 000458 - PSTH Neurosift Exploration + +See [001_summarize_contents.ipynb](./001_summarize_contents.ipynb) for a summary of the contents of the NWB files in this Dandiset, including the trial information. You can use that to decide which NWB files to examine. Here's an example of one that targets the motor cortex: + +``` +Asset 7: sub-551397/sub-551397_ses-20210211_behavior+ecephys.nwb -- size: 15202.44 MB +in vivo electrophysiology in a head-fixed mouse during cortical electrical microstimulation +EEG and Neuropixels recording during wakefulness and isoflurane anesthesia +single pulse electrical stimuli targeted to MOs (deep layers) + 1800 trials + BEHAVIORAL EPOCHS + awake: 386 valid and not running + isoflurane: 480 valid and not running + recovery: 553 valid and not running + STIMULUS TYPES + electrical: 1103 valid and not running + visual: 316 valid and not running + STIMULUS DESCRIPTIONS + biphasic: 1103 valid and not running + white circle: 316 valid and not running + ESTIM TARGET REGIONS + MOs: 1103 valid and not running + n/a: 316 valid and not running +``` + +First navigate to this Dandiset on DANDI Archive: [000458](https://dandiarchive.org/dandiset/000458). On the right panel, click on "Files" and then find [subject 551397](https://dandiarchive.org/dandiset/000458/0.230317.0039/files?location=sub-551397&page=1). For that NWB file, click on "OPEN WITH -> Neurosift". That should bring you [here](https://neurosift.app/?p=/nwb&url=https://api.dandiarchive.org/api/assets/d966b247-2bac-4ef0-8b80-aae010f50c98/download/&dandisetId=000458&dandisetVersion=0.230317.0039). + +* In Neurosift, open the "Intervals" panel and then click "PSTH" for the "trials" table. +* Select "Group trials by: behavioral_epoch" and "Sort units by: location". +* For trials table, enter `is_valid === 1 && is_running === 0 && stimulus_type === "electrical"`. This will restrict the trials to those that are valid, not running, and have an electrical stimulus. +* Scroll down on the units table and click to select some units in the MOs region. +* The raster plots show the spike trains for the unit where the Y-axis is the trial number and the X-axis is time aligned to the stimulus onset. +* Unit 500 shows a clear pattern of decreased activity following the stimulus for around 0.15 seconds and then a sharp increase in activity for the "awake" trials. +* Try adjusting the trials filter to have stimulus_type === "visual" instead of "electrical" to see that the pattern is not present for visual stimuli. +* Try selecting units at other locations. +* You can also adjust other settings. +* To share your view, use the "Share this tab" on the left panel of Neurosift. [Here's a link with the predefined state intact](https://neurosift.app/?p=/nwb&url=https://api.dandiarchive.org/api/assets/d966b247-2bac-4ef0-8b80-aae010f50c98/download/&dandisetId=000458&dandisetVersion=0.230317.0039&tab=view:PSTH|/intervals/trials^/units&tab-state=H4sIAAAAAAAAA03SvW7bMBSG4VsxOGRSC%2FE3dgAvGYJk7d%2BSFIYiMRIBiTREKkEQ%2BN77SkGLDt%2Bgw8f00Qd9iOxH3xbf%2FYyhPHRZ3Dyag6zMQRFNDLHEkWuyJ4fK1jWRRBFNDLHEkWuyJziJkziJkziJkziJkziJkziFUziFUziFUziFUziFUziN0ziN0ziN0ziN0ziN0ziDMziDMziDMziDMziDMziLsziLsziLsziLsziLsziHcziHcziHc7g953vOD5zTm6U3S2%2BW3iy9WXqz9Oboy9GVoytXm9%2BVaMbQxx%2FpVzOH5nn0a%2FEil2YupxImLxD9nJbz7ftfIW7Esx%2Ba15DmZjz5c2oHUYnC4Zjvwlj8jAj59MrN3e54PO7k7upqx2ReYgyx32b1Osv8xTIu%2BVTez34bP4ntM5hD24xPgntzmsv6SeT%2FF3gUY2qbElJENLldt3wLsUtv35rY%2B%2B%2BFFT4%2B34JdvtRfLc7HjgcpLpU4z%2F4lb2RYf5K3ncu8%2BGqb3Idc%2Fj1PKZXBd5%2BzF96R4eBDP6xXT74Ly8TlcZluQ%2BROXV8ufwD179KNzQIAAA%3D%3D). + +Now you can explore the other NWB files in this Dandiset! \ No newline at end of file diff --git a/000458/FlatironInstitute/README.md b/000458/FlatironInstitute/README.md new file mode 100644 index 0000000..6ea29d2 --- /dev/null +++ b/000458/FlatironInstitute/README.md @@ -0,0 +1,7 @@ +# Exploration of Dandiset 000458 + +* [000_lindi_vs_fsspec_streaming.ipynb](./000_lindi_vs_fsspec_streaming.ipynb): Timing comparison of two methods of streaming NWB files for this Dandiset. +* [001_summarize_contents.ipynb](./001_summarize_contents.ipynb): Summarize the contents of the NWB files in this Dandiset. +* [PSTH_neurosift_exploration.md](./PSTH_neurosift_exploration.md): Exploration of the PSTH data in this Dandiset using Neurosift. + +See also [../AllenInstitute/README.md](../AllenInstitute/README.md). diff --git a/000458/FlatironInstitute/environment.yaml b/000458/FlatironInstitute/environment.yaml new file mode 100644 index 0000000..470a2aa --- /dev/null +++ b/000458/FlatironInstitute/environment.yaml @@ -0,0 +1,16 @@ +# run: conda env create --file environment.yaml +# activate: conda activate 000458_fi +# update: conda env update --file environment.yaml --prune +name: 000458_fi +channels: + - conda-forge +dependencies: + - python==3.12 + - pip + - pip: + - dandi + - pynwb + - ipykernel + - matplotlib + - lindi==0.4.0a1 + - fsspec \ No newline at end of file diff --git a/000458/FlatironInstitute/helpers/stream_nwbfile.py b/000458/FlatironInstitute/helpers/stream_nwbfile.py new file mode 100644 index 0000000..2eacf7f --- /dev/null +++ b/000458/FlatironInstitute/helpers/stream_nwbfile.py @@ -0,0 +1,35 @@ +# See 000_lindi_vs_fsspec_streaming.py for why lindi is used rather than fsspec. + +from pynwb import NWBHDF5IO +from dandi.dandiapi import DandiAPIClient +import lindi + + +def stream_nwbfile(DANDISET_ID, file_path): + '''Stream NWB file from DANDI archive. + + Parameters + ---------- + DANDISET_ID : str + Dandiset ID + file_path : str + Path to NWB file in DANDI archive + + Returns + ------- + nwbfile : NWBFile + NWB file + io : NWBHDF5IO + NWB IO object (for closing) + + Notes + ----- + The io object must be closed after use. + ''' + with DandiAPIClient() as client: + asset = client.get_dandiset(DANDISET_ID, 'draft').get_asset_by_path(file_path) + asset_url = asset.get_content_url(follow_redirects=0, strip_query=True) + file = lindi.LindiH5pyFile.from_hdf5_file(asset_url) + io = NWBHDF5IO(file=file, load_namespaces=True) + nwbfile = io.read() + return nwbfile, io