Add support for NWB schemas v2.8.0 (#630)

* Add nwb 2.8.0 schemas and regenerated types

* Update requirements to use pynwb and nwbinspector from github@dev

* ...

* Fix typo of name for dataset in bands table in ecephys tutorial

* debug unexpected pynwb version

* Remove hdmf-zarr dependency, as it is not used

* Fix: Make inherited read-only datasets in schema read-only properties in matnwb

* Add debug flag for printing installed python packages in PynwbTutorialTest

* Add more detailed device example to ecephys tutorial

* Add more detailed device example to ophys tutorial

+tests/+unit/PynwbTutorialTest.m

@@ -25,19 +25,22 @@
             'streaming.py', ...             % Requires that HDF5 library is installed with the ROS3 driver enabled which is not a given
             'object_id.py', ...             % Does not export nwb file
             'plot_configurator.py', ...     % Does not export nwb file
+            'plot_zarr_io', ...             % Does not export nwb file in nwb format
             'brain_observatory.py', ...     % Requires allen sdk
             'extensions.py'};               % Discrepancy between tutorial and schema: https://github.com/NeurodataWithoutBorders/pynwb/issues/1952
 
         % SkippedFiles - Name of exported nwb files to skip reading with matnwb
         SkippedFiles = {'family_nwb_file_0.nwb'} % requires family driver from h5py
 
         % PythonDependencies - Package dependencies for running pynwb tutorials
-        PythonDependencies = {'hdmf-zarr', 'dataframe-image', 'matplotlib'}
+        PythonDependencies = {'dataframe-image', 'matplotlib'}
     end
 
     properties (Access = private)
         PythonEnvironment % Stores the value of the environment variable 
         % "PYTHONPATH" to restore when test is finished.
+
+        Debug (1,1) logical = false
     end
 
     methods (TestClassSetup)
@@ -66,6 +69,12 @@ function setupClass(testCase)
             L = dir('temp_venv/lib/python*/site-*'); % Find the site-packages folder
             pythonPath = fullfile(L.folder, L.name);
             setenv('PYTHONPATH', pythonPath)
+
+            pythonPath = tests.util.getPythonPath();
+
+            if testCase.Debug
+                [~, m] = system(sprintf('%s -m pip list', pythonPath)); disp(m)
+            end
         end
     end
 
@@ -172,7 +181,12 @@ function installPythonDependencies(testCase)
             for i = 1:numel(testCase.PythonDependencies)
                 iName = testCase.PythonDependencies{i};
                 installCmdStr = sprintf('%s install %s', pipExecutable, iName);
-                evalc( "system(installCmdStr)" ); % Install without command window output
+
+                if testCase.Debug
+                    [~, m] = system(installCmdStr); disp(m)
+                else
+                    evalc( "system(installCmdStr)" ); % Install without command window output
+                end
             end
         end
     end
@@ -208,7 +222,7 @@ function installPythonDependencies(testCase)
 end
 
 function pynwbFolder = downloadPynwb()
-    githubUrl = 'https://github.com/NeurodataWithoutBorders/pynwb/archive/refs/heads/master.zip';
+    githubUrl = 'https://github.com/NeurodataWithoutBorders/pynwb/archive/refs/heads/dev.zip';
     pynwbFolder = downloadZippedGithubRepo(githubUrl, '.'); % Download in current directory
 end
 

+tests/requirements.txt

@@ -1,3 +1,3 @@
-pynwb
 hdf5plugin
-nwbinspector
+git+https://github.com/NeurodataWithoutBorders/nwbinspector.git@dev
+git+https://github.com/NeurodataWithoutBorders/pynwb.git@dev

+types/+core/Device.m

@@ -4,8 +4,11 @@
 
 % OPTIONAL PROPERTIES
 properties
-    description; %  (char) Description of the device (e.g., model, firmware version, processing software version, etc.) as free-form text.
-    manufacturer; %  (char) The name of the manufacturer of the device.
+    description; %  (char) Description of the device as free-form text. If there is any software/firmware associated with the device, the names and versions of those can be added to NWBFile.was_generated_by.
+    manufacturer; %  (char) The name of the manufacturer of the device, e.g., Imec, Plexon, Thorlabs.
+    model_name; %  (char) The model name of the device, e.g., Neuropixels 1.0, V-Probe, Bergamo III.
+    model_number; %  (char) The model number (or part/product number) of the device, e.g., PRB_1_4_0480_1, PLX-VP-32-15SE(75)-(260-80)(460-10)-300-(1)CON/32m-V, BERGAMO.
+    serial_number; %  (char) The serial number of the device.
 end
 
 methods
@@ -20,9 +23,15 @@
         p.StructExpand = false;
         addParameter(p, 'description',[]);
         addParameter(p, 'manufacturer',[]);
+        addParameter(p, 'model_name',[]);
+        addParameter(p, 'model_number',[]);
+        addParameter(p, 'serial_number',[]);
         misc.parseSkipInvalidName(p, varargin);
         obj.description = p.Results.description;
         obj.manufacturer = p.Results.manufacturer;
+        obj.model_name = p.Results.model_name;
+        obj.model_number = p.Results.model_number;
+        obj.serial_number = p.Results.serial_number;
         if strcmp(class(obj), 'types.core.Device')
             cellStringArguments = convertContainedStringsToChars(varargin(1:2:end));
             types.util.checkUnset(obj, unique(cellStringArguments));
@@ -35,6 +44,15 @@
     function set.manufacturer(obj, val)
         obj.manufacturer = obj.validate_manufacturer(val);
     end
+    function set.model_name(obj, val)
+        obj.model_name = obj.validate_model_name(val);
+    end
+    function set.model_number(obj, val)
+        obj.model_number = obj.validate_model_number(val);
+    end
+    function set.serial_number(obj, val)
+        obj.serial_number = obj.validate_serial_number(val);
+    end
     %% VALIDATORS
 
     function val = validate_description(obj, val)
@@ -73,6 +91,60 @@
         validshapes = {[1]};
         types.util.checkDims(valsz, validshapes);
     end
+    function val = validate_model_name(obj, val)
+        val = types.util.checkDtype('model_name', 'char', val);
+        if isa(val, 'types.untyped.DataStub')
+            if 1 == val.ndims
+                valsz = [val.dims 1];
+            else
+                valsz = val.dims;
+            end
+        elseif istable(val)
+            valsz = [height(val) 1];
+        elseif ischar(val)
+            valsz = [size(val, 1) 1];
+        else
+            valsz = size(val);
+        end
+        validshapes = {[1]};
+        types.util.checkDims(valsz, validshapes);
+    end
+    function val = validate_model_number(obj, val)
+        val = types.util.checkDtype('model_number', 'char', val);
+        if isa(val, 'types.untyped.DataStub')
+            if 1 == val.ndims
+                valsz = [val.dims 1];
+            else
+                valsz = val.dims;
+            end
+        elseif istable(val)
+            valsz = [height(val) 1];
+        elseif ischar(val)
+            valsz = [size(val, 1) 1];
+        else
+            valsz = size(val);
+        end
+        validshapes = {[1]};
+        types.util.checkDims(valsz, validshapes);
+    end
+    function val = validate_serial_number(obj, val)
+        val = types.util.checkDtype('serial_number', 'char', val);
+        if isa(val, 'types.untyped.DataStub')
+            if 1 == val.ndims
+                valsz = [val.dims 1];
+            else
+                valsz = val.dims;
+            end
+        elseif istable(val)
+            valsz = [height(val) 1];
+        elseif ischar(val)
+            valsz = [size(val, 1) 1];
+        else
+            valsz = size(val);
+        end
+        validshapes = {[1]};
+        types.util.checkDims(valsz, validshapes);
+    end
     %% EXPORT
     function refs = export(obj, fid, fullpath, refs)
         refs = export@types.core.NWBContainer(obj, fid, fullpath, refs);
@@ -85,6 +157,15 @@
         if ~isempty(obj.manufacturer)
             io.writeAttribute(fid, [fullpath '/manufacturer'], obj.manufacturer);
         end
+        if ~isempty(obj.model_name)
+            io.writeAttribute(fid, [fullpath '/model_name'], obj.model_name);
+        end
+        if ~isempty(obj.model_number)
+            io.writeAttribute(fid, [fullpath '/model_number'], obj.model_number);
+        end
+        if ~isempty(obj.serial_number)
+            io.writeAttribute(fid, [fullpath '/serial_number'], obj.serial_number);
+        end
     end
 end
 

+types/+core/EventWaveform.m

@@ -1,5 +1,5 @@
 classdef EventWaveform < types.core.NWBDataInterface & types.untyped.GroupClass
-% EVENTWAVEFORM Represents either the waveforms of detected events, as extracted from a raw data trace in /acquisition, or the event waveforms that were stored during experiment acquisition.
+% EVENTWAVEFORM DEPRECATED. Represents either the waveforms of detected events, as extracted from a raw data trace in /acquisition, or the event waveforms that were stored during experiment acquisition.
 
 
 % OPTIONAL PROPERTIES

+types/+core/ImageMaskSeries.m

@@ -1,5 +1,5 @@
 classdef ImageMaskSeries < types.core.ImageSeries & types.untyped.GroupClass
-% IMAGEMASKSERIES An alpha mask that is applied to a presented visual stimulus. The 'data' array contains an array of mask values that are applied to the displayed image. Mask values are stored as RGBA. Mask can vary with time. The timestamps array indicates the starting time of a mask, and that mask pattern continues until it's explicitly changed.
+% IMAGEMASKSERIES DEPRECATED. An alpha mask that is applied to a presented visual stimulus. The 'data' array contains an array of mask values that are applied to the displayed image. Mask values are stored as RGBA. Mask can vary with time. The timestamps array indicates the starting time of a mask, and that mask pattern continues until it's explicitly changed.
 
 
 % OPTIONAL PROPERTIES

+types/+core/NWBFile.m

@@ -50,6 +50,7 @@
     general_subject; %  (Subject) Information about the animal or person from which the data was measured.
     general_surgery; %  (char) Narrative description about surgery/surgeries, including date(s) and who performed surgery.
     general_virus; %  (char) Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.
+    general_was_generated_by; %  (char) Name and version of software package(s) used to generate data contained in this NWB File. For each software package or library, include the name of the software as the first value and the version as the second value.
     intervals; %  (TimeIntervals) Optional additional table(s) for describing other experimental time intervals.
     intervals_epochs; %  (TimeIntervals) Divisions in time marking experimental stages or sub-divisions of a single recording session.
     intervals_invalid_times; %  (TimeIntervals) Time intervals that should be removed from analysis.
@@ -64,7 +65,7 @@
 methods
     function obj = NWBFile(varargin)
         % NWBFILE Constructor for NWBFile
-        varargin = [{'nwb_version' '2.7.0'} varargin];
+        varargin = [{'nwb_version' '2.8.0'} varargin];
         obj = obj@types.core.NWBContainer(varargin{:});
 
 
@@ -107,6 +108,7 @@
         addParameter(p, 'general_subject',[]);
         addParameter(p, 'general_surgery',[]);
         addParameter(p, 'general_virus',[]);
+        addParameter(p, 'general_was_generated_by',[]);
         addParameter(p, 'identifier',[]);
         addParameter(p, 'intervals',types.untyped.Set());
         addParameter(p, 'intervals_epochs',[]);
@@ -157,6 +159,7 @@
         obj.general_subject = p.Results.general_subject;
         obj.general_surgery = p.Results.general_surgery;
         obj.general_virus = p.Results.general_virus;
+        obj.general_was_generated_by = p.Results.general_was_generated_by;
         obj.identifier = p.Results.identifier;
         obj.intervals = p.Results.intervals;
         obj.intervals_epochs = p.Results.intervals_epochs;
@@ -282,6 +285,9 @@
     function set.general_virus(obj, val)
         obj.general_virus = obj.validate_general_virus(val);
     end
+    function set.general_was_generated_by(obj, val)
+        obj.general_was_generated_by = obj.validate_general_was_generated_by(val);
+    end
     function set.identifier(obj, val)
         obj.identifier = obj.validate_identifier(val);
     end
@@ -727,6 +733,24 @@
         validshapes = {[1]};
         types.util.checkDims(valsz, validshapes);
     end
+    function val = validate_general_was_generated_by(obj, val)
+        val = types.util.checkDtype('general_was_generated_by', 'char', val);
+        if isa(val, 'types.untyped.DataStub')
+            if 1 == val.ndims
+                valsz = [val.dims 1];
+            else
+                valsz = val.dims;
+            end
+        elseif istable(val)
+            valsz = [height(val) 1];
+        elseif ischar(val)
+            valsz = [size(val, 1) 1];
+        else
+            valsz = size(val);
+        end
+        validshapes = {[2,Inf]};
+        types.util.checkDims(valsz, validshapes);
+    end
     function val = validate_identifier(obj, val)
         val = types.util.checkDtype('identifier', 'char', val);
         if isa(val, 'types.untyped.DataStub')
@@ -1041,6 +1065,14 @@
                 io.writeDataset(fid, [fullpath '/general/virus'], obj.general_virus);
             end
         end
+        io.writeGroup(fid, [fullpath '/general']);
+        if ~isempty(obj.general_was_generated_by)
+            if startsWith(class(obj.general_was_generated_by), 'types.untyped.')
+                refs = obj.general_was_generated_by.export(fid, [fullpath '/general/was_generated_by'], refs);
+            elseif ~isempty(obj.general_was_generated_by)
+                io.writeDataset(fid, [fullpath '/general/was_generated_by'], obj.general_was_generated_by, 'forceArray');
+            end
+        end
         if startsWith(class(obj.identifier), 'types.untyped.')
             refs = obj.identifier.export(fid, [fullpath '/identifier'], refs);
         elseif ~isempty(obj.identifier)

+types/+core/SpikeEventSeries.m

@@ -1,5 +1,5 @@
 classdef SpikeEventSeries < types.core.ElectricalSeries & types.untyped.GroupClass
-% SPIKEEVENTSERIES Stores snapshots/snippets of recorded spike events (i.e., threshold crossings). This may also be raw data, as reported by ephys hardware. If so, the TimeSeries::description field should describe how events were detected. All SpikeEventSeries should reside in a module (under EventWaveform interface) even if the spikes were reported and stored by hardware. All events span the same recording channels and store snapshots of equal duration. TimeSeries::data array structure: [num events] [num channels] [num samples] (or [num events] [num samples] for single electrode).
+% SPIKEEVENTSERIES Stores snapshots/snippets of recorded spike events (i.e., threshold crossings). This may also be raw data, as reported by ephys hardware. If so, the TimeSeries::description field should describe how events were detected. All events span the same recording channels and store snapshots of equal duration. TimeSeries::data array structure: [num events] [num channels] [num samples] (or [num events] [num samples] for single electrode).
 
 
 

+types/+core/Units.m

@@ -13,9 +13,9 @@
     spike_times_index; %  (VectorIndex) Index into the spike_times dataset.
     waveform_mean; %  (VectorData) Spike waveform mean for each spike unit.
     waveform_sd; %  (VectorData) Spike waveform standard deviation for each spike unit.
-    waveforms; %  (VectorData) Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be in the same order as the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.
-    waveforms_index; %  (VectorIndex) Index into the waveforms dataset. One value for every spike event. See 'waveforms' for more detail.
-    waveforms_index_index; %  (VectorIndex) Index into the waveforms_index dataset. One value for every unit (row in the table). See 'waveforms' for more detail.
+    waveforms; %  (VectorData) Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be the same as the order of the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.
+    waveforms_index; %  (VectorIndex) Index into the 'waveforms' dataset. One value for every spike event. See 'waveforms' for more detail.
+    waveforms_index_index; %  (VectorIndex) Index into the 'waveforms_index' dataset. One value for every unit (row in the table). See 'waveforms' for more detail.
 end
 
 methods

.github/workflows/run_tests.yml

@@ -28,6 +28,7 @@ jobs:
         run: |
           python -m pip install -U pip
           pip install -r +tests/requirements.txt
+          python -m pip list
           echo "HDF5_PLUGIN_PATH=$(python -c "import hdf5plugin; print(hdf5plugin.PLUGINS_PATH)")" >> "$GITHUB_ENV"
           echo $( python -m pip show nwbinspector | grep ^Location: | awk '{print $2}' )
       - name: Install MATLAB