Support NWB schema 2.2.2 (#1146)

Update PyNWB API to support NWB schema 2.2.2 (and 2.2.0 and 2.2.1) as well as HDMF 1.6.1

README.rst

@@ -93,7 +93,7 @@ You are under no obligation whatsoever to provide any bug fixes, patches, or upg
 COPYRIGHT
 =========
 
-"pynwb" Copyright (c) 2017-2019, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy).  All rights reserved.
+"pynwb" Copyright (c) 2017-2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy).  All rights reserved.
 If you have questions about your rights to use or distribute this software, please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov.
 
 NOTICE.  This Software was developed under funding from the U.S. Department of Energy and the U.S. Government consequently retains certain rights. As such, the U.S. Government has been granted for itself and others acting on its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the Software to reproduce, distribute copies to the public, prepare derivative works, and perform publicly and display publicly, and to permit other to do so.

requirements-min.txt

@@ -1,6 +1,6 @@
 # these minimum requirements specify '==' for testing; setup.py replaces '==' with '>='
 h5py==2.9  # support for setting attrs to lists of utf-8 added in 2.9
-hdmf==1.5.4,<2
+hdmf==1.6.1,<2
 numpy==1.16
 pandas==0.23
 python-dateutil==2.7

requirements.txt

@@ -1,5 +1,5 @@
 h5py==2.10.0
-hdmf==1.5.4
+hdmf==1.6.1
 numpy==1.18.1
 pandas==0.25.3
-python-dateutil==2.8.0
+python-dateutil==2.8.1

src/pynwb/__init__.py

@@ -168,8 +168,8 @@ def get_class(**kwargs):
     specification. If you want to define a custom mapping, you should not use this function and you should define the
     class manually.
 
-    Examples
-    --------
+    Examples:
+
     Generating and registering an extension is as simple as::
 
         MyClass = get_class('MyClass', 'ndx-my-extension')

src/pynwb/base.py

@@ -232,10 +232,14 @@ def time_unit(self):
 
 @register_class('Image', CORE_NAMESPACE)
 class Image(NWBData):
+    """
+    Abstract image class. It is recommended to instead use pynwb.image.GrayscaleImage or pynwb.image.RGPImage where
+    appropriate.
+    """
     __nwbfields__ = ('data', 'resolution', 'description')
 
     @docval({'name': 'name', 'type': str, 'doc': 'The name of this TimeSeries dataset'},
-            {'name': 'data', 'type': ('array_data', 'data'), 'doc': 'data of image',
+            {'name': 'data', 'type': ('array_data', 'data'), 'doc': 'data of image. Dimensions: x, y [, r,g,b[,a]]',
              'shape': ((None, None), (None, None, 3), (None, None, 4))},
             {'name': 'resolution', 'type': 'float', 'doc': 'pixels / cm', 'default': None},
             {'name': 'description', 'type': str, 'doc': 'description of image', 'default': None})

src/pynwb/core.py

@@ -5,7 +5,8 @@
 from hdmf.utils import docval, getargs, ExtenderMeta, call_docval_func, popargs, get_docval, fmt_docval_args
 from hdmf import Container, Data, DataRegion, get_region_slicer
 from hdmf.container import AbstractContainer
-from hdmf.common import DynamicTable, DynamicTableRegion  # NOQA
+from hdmf.common import DynamicTable, DynamicTableRegion  # noqa: F401
+from hdmf.common import VectorData, VectorIndex, ElementIdentifiers  # noqa: F401
 
 from . import CORE_NAMESPACE, register_class
 

src/pynwb/ecephys.py

@@ -18,18 +18,25 @@ class ElectrodeGroup(NWBContainer):
     __nwbfields__ = ('name',
                      'description',
                      'location',
-                     'device')
+                     'device',
+                     'position')
 
     @docval({'name': 'name', 'type': str, 'doc': 'the name of this electrode'},
             {'name': 'description', 'type': str, 'doc': 'description of this electrode group'},
             {'name': 'location', 'type': str, 'doc': 'description of location of this electrode group'},
-            {'name': 'device', 'type': Device, 'doc': 'the device that was used to record from this electrode group'})
+            {'name': 'device', 'type': Device, 'doc': 'the device that was used to record from this electrode group'},
+            {'name': 'position', 'type': 'array_data',
+             'doc': 'stereotaxic position of this electrode group (x, y, z)', 'default': None})
     def __init__(self, **kwargs):
         call_docval_func(super(ElectrodeGroup, self).__init__, kwargs)
-        description, location, device = popargs("description", "location", "device", kwargs)
+        description, location, device, position = popargs('description', 'location', 'device', 'position', kwargs)
         self.description = description
         self.location = location
         self.device = device
+        if position and len(position) != 3:
+            raise Exception('ElectrodeGroup position argument must have three elements: x, y, z, but received: %s'
+                            % position)
+        self.position = position
 
 
 @register_class('ElectricalSeries', CORE_NAMESPACE)

src/pynwb/file.py

@@ -493,14 +493,19 @@ def add_electrode_column(self, **kwargs):
         self.__check_electrodes()
         call_docval_func(self.electrodes.add_column, kwargs)
 
-    @docval({'name': 'x', 'type': 'float', 'doc': 'the x coordinate of the position'},
-            {'name': 'y', 'type': 'float', 'doc': 'the y coordinate of the position'},
-            {'name': 'z', 'type': 'float', 'doc': 'the z coordinate of the position'},
+    @docval({'name': 'x', 'type': 'float', 'doc': 'the x coordinate of the position (+x is posterior)'},
+            {'name': 'y', 'type': 'float', 'doc': 'the y coordinate of the position (+y is inferior)'},
+            {'name': 'z', 'type': 'float', 'doc': 'the z coordinate of the position (+z is right)'},
             {'name': 'imp', 'type': 'float', 'doc': 'the impedance of the electrode'},
             {'name': 'location', 'type': str, 'doc': 'the location of electrode within the subject e.g. brain region'},
             {'name': 'filtering', 'type': str, 'doc': 'description of hardware filtering'},
             {'name': 'group', 'type': ElectrodeGroup, 'doc': 'the ElectrodeGroup object to add to this NWBFile'},
             {'name': 'id', 'type': int, 'doc': 'a unique identifier for the electrode', 'default': None},
+            {'name': 'rel_x', 'type': 'float', 'doc': 'the x coordinate within the electrode group', 'default': None},
+            {'name': 'rel_y', 'type': 'float', 'doc': 'the y coordinate within the electrode group', 'default': None},
+            {'name': 'rel_z', 'type': 'float', 'doc': 'the z coordinate within the electrode group', 'default': None},
+            {'name': 'reference', 'type': str, 'doc': 'Description of the reference used for this electrode.',
+             'default': None},
             allow_extra=True)
     def add_electrode(self, **kwargs):
         """
@@ -515,6 +520,17 @@ def add_electrode(self, **kwargs):
         d = _copy.copy(kwargs['data']) if kwargs.get('data') is not None else kwargs
         if d.get('group_name', None) is None:
             d['group_name'] = d['group'].name
+
+        new_cols = [('rel_x', 'the x coordinate within the electrode group'),
+                    ('rel_y', 'the y coordinate within the electrode group'),
+                    ('rel_z', 'the z coordinate within the electrode group'),
+                    ('reference', 'Description of the reference used for this electrode.')]
+        for col_name, col_doc in new_cols:
+            if kwargs[col_name] is not None and col_name not in self.electrodes:
+                self.electrodes.add_column(col_name, col_doc)
+            else:
+                d.pop(col_name)  # remove args from d if not set
+
         call_docval_func(self.electrodes.add_row, d)
 
     @docval({'name': 'region', 'type': (slice, list, tuple), 'doc': 'the indices of the table'},
@@ -757,7 +773,8 @@ def ElectrodeTable(name='electrodes',
                        ('location', 'the location of channel within the subject e.g. brain region'),
                        ('filtering', 'description of hardware filtering'),
                        ('group', 'a reference to the ElectrodeGroup this electrode is a part of'),
-                       ('group_name', 'the name of the ElectrodeGroup this electrode is a part of')]
+                       ('group_name', 'the name of the ElectrodeGroup this electrode is a part of')
+                       ]
                       )
 
 

src/pynwb/image.py

@@ -21,7 +21,8 @@ class ImageSeries(TimeSeries):
 
     @docval(*get_docval(TimeSeries.__init__, 'name'),  # required
             {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': ([None] * 3, [None] * 4),
-             'doc': 'The data this TimeSeries dataset stores. Can also store binary data e.g. image frames',
+             'doc': 'The data this TimeSeries dataset stores. Can also store binary data e.g. image frames. '
+                    'dimensions: time, x, y [, z]',
              'default': None},
             *get_docval(TimeSeries.__init__, 'unit'),
             {'name': 'format', 'type': str,
@@ -130,7 +131,9 @@ class OpticalSeries(ImageSeries):
                      'field_of_view',
                      'orientation')
 
-    @docval(*get_docval(ImageSeries.__init__, 'name', 'data'),  # required
+    @docval(*get_docval(ImageSeries.__init__, 'name'),
+            {'name': 'data', 'type': ('array_data', 'data'), 'shape': ([None] * 3, [None, None, None, 3]),
+             'doc': 'Images presented to subject, either grayscale or RGB'},
             *get_docval(ImageSeries.__init__, 'unit', 'format'),
             {'name': 'distance', 'type': 'float', 'doc': 'Distance from camera/monitor to target/eye.'},  # required
             {'name': 'field_of_view', 'type': ('array_data', 'data', 'TimeSeries'), 'shape': ((2, ), (3, )),  # required

src/pynwb/io/core.py

@@ -1,9 +1,14 @@
 from hdmf.build import ObjectMapper, RegionBuilder
+from hdmf.common import VectorData
+from hdmf.utils import getargs, docval
+from hdmf.spec import AttributeSpec
+from hdmf.build import BuildManager
 
 from .. import register_map
 
 from pynwb.file import NWBFile
 from pynwb.core import NWBData, NWBContainer
+from pynwb.misc import Units
 
 
 class NWBBaseTypeMapper(ObjectMapper):
@@ -19,7 +24,6 @@ def get_nwb_file(container):
 
 @register_map(NWBContainer)
 class NWBContainerMapper(NWBBaseTypeMapper):
-
     pass
 
 
@@ -46,3 +50,27 @@ def carg_region(self, builder, manager):
         if not isinstance(builder.data, RegionBuilder):
             raise ValueError("'builder' must be a RegionBuilder")
         return builder.data.region
+
+
+@register_map(VectorData)
+class VectorDataMap(ObjectMapper):
+
+    @docval({"name": "spec", "type": AttributeSpec, "doc": "the spec to get the attribute value for"},
+            {"name": "container", "type": VectorData, "doc": "the container to get the attribute value from"},
+            {"name": "manager", "type": BuildManager, "doc": "the BuildManager used for managing this build"},
+            returns='the value of the attribute')
+    def get_attr_value(self, **kwargs):
+        ''' Get the value of the attribute corresponding to this spec from the given container '''
+        spec, container, manager = getargs('spec', 'container', 'manager', kwargs)
+
+        # handle custom mapping of container Units.waveform_rate -> spec Units.waveform_mean.sampling_rate
+        if isinstance(container.parent, Units):
+            if container.name == 'waveform_mean' or container.name == 'waveform_sd':
+                if spec.name == 'sampling_rate':
+                    return container.parent.waveform_rate
+                if spec.name == 'unit':
+                    return container.parent.waveform_unit
+            if container.name == 'spike_times':
+                if spec.name == 'resolution':
+                    return container.parent.resolution
+        return super().get_attr_value(**kwargs)

src/pynwb/io/misc.py

@@ -7,6 +7,38 @@
 @register_map(Units)
 class UnitsMap(DynamicTableMap):
 
+    def __init__(self, spec):
+        super().__init__(spec)
+
+    @DynamicTableMap.constructor_arg('resolution')
+    def resolution_carg(self, builder, manager):
+        if 'spike_times' in builder:
+            return builder['spike_times'].attributes.get('resolution')
+        return None
+
+    @DynamicTableMap.constructor_arg('waveform_rate')
+    def waveform_rate_carg(self, builder, manager):
+        return self._get_waveform_stat(builder, 'sampling_rate')
+
+    @DynamicTableMap.constructor_arg('waveform_unit')
+    def waveform_unit_carg(self, builder, manager):
+        return self._get_waveform_stat(builder, 'unit')
+
+    def _get_waveform_stat(self, builder, attribute):
+        if 'waveform_mean' not in builder and 'waveform_sd' not in builder:
+            return None
+        mean_stat = None
+        sd_stat = None
+        if 'waveform_mean' in builder:
+            mean_stat = builder['waveform_mean'].attributes.get(attribute)
+        if 'waveform_sd' in builder:
+            sd_stat = builder['waveform_sd'].attributes.get(attribute)
+        if mean_stat is not None and sd_stat is not None:
+            if mean_stat != sd_stat:
+                # throw warning
+                pass
+        return mean_stat
+
     @DynamicTableMap.object_attr("electrodes")
     def electrodes_column(self, container, manager):
         ret = container.get('electrodes')

src/pynwb/io/ophys.py

@@ -21,5 +21,10 @@ class ImagingPlaneMap(NWBContainerMapper):
     def __init__(self, spec):
         super(ImagingPlaneMap, self).__init__(spec)
         manifold_spec = self.spec.get_dataset('manifold')
+        origin_coords_spec = self.spec.get_dataset('origin_coords')
+        grid_spacing_spec = self.spec.get_dataset('grid_spacing')
         self.map_spec('unit', manifold_spec.get_attribute('unit'))
         self.map_spec('conversion', manifold_spec.get_attribute('conversion'))
+        self.map_spec('origin_coords_unit', origin_coords_spec.get_attribute('unit'))
+        self.map_spec('grid_spacing_unit', grid_spacing_spec.get_attribute('unit'))
+        self.map_spec('optical_channel', self.spec.get_neurodata_type('OpticalChannel'))

src/pynwb/io/retinotopy.py

@@ -0,0 +1,14 @@
+# from hdmf.build import ObjectMapper
+# from .. import register_map
+# from pynwb.retinotopy import ImagingRetinotopy
+#
+#
+# @register_map(ImagingRetinotopy)
+# class ImagingRetinotopyMap(ObjectMapper):
+#
+#     def __init__(self, spec):
+#         super().__init__(spec)
+#
+#         datasets = ['sign_map', 'axis_1_phase_map']
+#         for dataset_name in datasets:
+#             self.map_spec(dataset_name, spec.get_dataset(dataset_name))

src/pynwb/misc.py

@@ -131,6 +131,12 @@ class Units(DynamicTable):
     Event times of observed units (e.g. cell, synapse, etc.).
     """
 
+    __fields__ = (
+        'waveform_rate',
+        'waveform_unit',
+        'resolution'
+    )
+
     __columns__ = (
         {'name': 'spike_times', 'description': 'the spike times for each unit', 'index': True},
         {'name': 'obs_intervals', 'description': 'the observation intervals for each unit',
@@ -146,14 +152,24 @@ class Units(DynamicTable):
             *get_docval(DynamicTable.__init__, 'id', 'columns', 'colnames'),
             {'name': 'description', 'type': str, 'doc': 'a description of what is in this table', 'default': None},
             {'name': 'electrode_table', 'type': DynamicTable,
-             'doc': 'the table that the *electrodes* column indexes', 'default': None})
+             'doc': 'the table that the *electrodes* column indexes', 'default': None},
+            {'name': 'waveform_rate', 'type': 'float',
+             'doc': 'Sampling rate of the waveform means', 'default': None},
+            {'name': 'waveform_unit', 'type': str,
+             'doc': 'Unit of measurement of the waveform means', 'default': 'volts'},
+            {'name': 'resolution', 'type': 'float',
+             'doc': 'The smallest possible difference between two spike times', 'default': None},
+            )
     def __init__(self, **kwargs):
         if kwargs.get('description', None) is None:
             kwargs['description'] = "data on spiking units"
         call_docval_func(super(Units, self).__init__, kwargs)
         if 'spike_times' not in self.colnames:
             self.__has_spike_times = False
         self.__electrode_table = getargs('electrode_table', kwargs)
+        self.waveform_rate = getargs('waveform_rate', kwargs)
+        self.waveform_unit = getargs('waveform_unit', kwargs)
+        self.resolution = getargs('resolution', kwargs)
 
     @docval({'name': 'spike_times', 'type': 'array_data', 'doc': 'the spike times for each unit',
              'default': None, 'shape': (None,)},