diff --git a/CHANGELOG.md b/CHANGELOG.md index 32d95f45b..51bdcf77f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,14 @@ ### Breaking changes: - ``SweepTable`` has been deprecated in favor of the new icephys metadata tables. Use of ``SweepTable`` - is still possible but no longer recommended. @oruebel (#1349) + is still possible but no longer recommended. @oruebel (#1349 +- ``TimeSeries.__init__`` now requires the ``data`` argument because the 'data' dataset is required by the schema. + If a ``TimeSeries`` is read without a value for ``data``, it will be set to a default value. For most + ``TimeSeries``, this is a 1-dimensional empty array with dtype uint8. For ``ImageSeries`` and + ``DecompositionSeries``, this is a 3-dimensional empty array with dtype uint8. @rly (#1274) +- ``TimeSeries.__init__`` now requires the ``unit`` argument because the 'unit' attribute is required by the schema. + If a ``TimeSeries`` is read without a value for ``unit``, it will be set to a default value. For most + ``TimeSeries``, this is "unknown". For ``IndexSeries``, this is "N/A" according to the NWB 2.4.0 schema. @rly (#1274) ### New features: - Added new intracellular electrophysiology hierarchical table structure from ndx-icephys-meta to NWB core. @@ -12,8 +19,8 @@ ``SimultaneousRecordingsTable``, ``SequentialRecordingsTable``, ``RepetitionsTable`` and ``ExperimentalConditionsTable`` as well as corresponding updates to ``NWBFile`` to support interaction with the new tables. @oruebel (#1349) -- Added support for nwb-schema 2.4.0. See [Release Notes](https://nwb-schema.readthedocs.io/en/latest/format_release_notes.html) - for more details. @oruebel (#1349) +- Added support for NWB 2.4.0. See [Release Notes](https://nwb-schema.readthedocs.io/en/latest/format_release_notes.html) + for more details. @oruebel, @rly (#1349) - Dropped Python 3.6 support, added Python 3.9 support. @rly (#1377) - Updated requirements to allow compatibility with HDMF 3 and h5py 3. @rly (#1377) @@ -37,8 +44,12 @@ - Enforced electrode ID uniqueness during insertion into table. @CodyCBakerPhD (#1344) - Fixed integration tests with invalid test data that will be caught by future hdmf validator version. @dsleiter, @rly (#1366, #1376) -- Fixed build warnings in docs @oruebel (#1380) -- Fix intersphinx links in docs for numpy @oruebel (#1386) +- Fixed build warnings in docs. @oruebel (#1380) +- Fix intersphinx links in docs for numpy. @oruebel (#1386) +- Previously, the ``data`` argument was required in ``OpticalSeries.__init__`` even though ``external_file`` could + be provided in place of ``data``. ``OpticalSeries.__init__`` now makes ``data`` optional. However, this has the + side effect of moving the position of ``data`` to later in the argument list, which may break code that relies + on positional arguments for ``OpticalSeries.__init__``. @rly (#1274) ## PyNWB 1.5.1 (May 24, 2021) diff --git a/docs/gallery/domain/ophys.py b/docs/gallery/domain/ophys.py index f457eb8d9..623b52e51 100644 --- a/docs/gallery/domain/ophys.py +++ b/docs/gallery/domain/ophys.py @@ -152,7 +152,13 @@ data = [0., 1., 2., 3., 4., 5., 6., 7., 8., 9.] timestamps = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] -rrs = fl.create_roi_response_series('my_rrs', data, rt_region, unit='lumens', timestamps=timestamps) +rrs = fl.create_roi_response_series( + name='my_rrs', + data=data, + rois=rt_region, + unit='lumens', + timestamps=timestamps +) #################### diff --git a/src/pynwb/base.py b/src/pynwb/base.py index 1a5ced830..9924bc7bf 100644 --- a/src/pynwb/base.py +++ b/src/pynwb/base.py @@ -1,11 +1,10 @@ from warnings import warn from collections.abc import Iterable +import numpy as np from typing import NamedTuple from hdmf.utils import docval, getargs, popargs, call_docval_func, get_docval from hdmf.common import DynamicTable, VectorData -import numpy as np - from . import register_class, CORE_NAMESPACE from .core import NWBDataInterface, MultiContainerInterface, NWBData @@ -99,12 +98,15 @@ class TimeSeries(NWBDataInterface): __time_unit = "seconds" + # values used when a TimeSeries is read and missing required fields + DEFAULT_DATA = np.ndarray(shape=(0, ), dtype=np.uint8) + DEFAULT_UNIT = 'unknown' + @docval({'name': 'name', 'type': str, 'doc': 'The name of this TimeSeries dataset'}, # required {'name': 'data', 'type': ('array_data', 'data', 'TimeSeries'), 'doc': ('The data values. The first dimension must be time. ' - 'Can also store binary data, e.g., image frames'), - 'default': None}, - {'name': 'unit', 'type': str, 'doc': 'The base unit of measurement (should be SI unit)', 'default': None}, + 'Can also store binary data, e.g., image frames')}, + {'name': 'unit', 'type': str, 'doc': 'The base unit of measurement (should be SI unit)'}, {'name': 'resolution', 'type': (str, 'float'), 'doc': 'The smallest meaningful difference (in specified unit) between values in data', 'default': -1.0}, {'name': 'conversion', 'type': (str, 'float'), diff --git a/src/pynwb/image.py b/src/pynwb/image.py index 666177296..ba26113a3 100644 --- a/src/pynwb/image.py +++ b/src/pynwb/image.py @@ -1,7 +1,8 @@ import warnings +import numpy as np from collections.abc import Iterable -from hdmf.utils import docval, popargs, call_docval_func, get_docval +from hdmf.utils import docval, getargs, popargs, call_docval_func, get_docval from . import register_class, CORE_NAMESPACE from .base import TimeSeries, Image @@ -21,18 +22,27 @@ class ImageSeries(TimeSeries): 'format', 'device') + # value used when an ImageSeries is read and missing data + DEFAULT_DATA = np.ndarray(shape=(0, 0, 0), dtype=np.uint8) + # TODO: copy new docs from 2.4 schema + @docval(*get_docval(TimeSeries.__init__, 'name'), # required {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': ([None] * 3, [None] * 4), 'doc': ('The data values. Can be 3D or 4D. The first dimension must be time (frame). The second and third ' - 'dimensions represent x and y. The optional fourth dimension represents z.'), + 'dimensions represent x and y. The optional fourth dimension represents z. Either data or ' + 'external_file must be specified (not None), but not both. If data is not specified, ' + 'data will be set to an empty 3D array.'), + 'default': None}, + {'name': 'unit', 'type': str, + 'doc': ('The unit of measurement of the image data, e.g., values between 0 and 255. Required when data ' + 'is specified. If unit (and data) are not specified, then unit will be set to "unknown".'), 'default': None}, - *get_docval(TimeSeries.__init__, 'unit'), {'name': 'format', 'type': str, 'doc': 'Format of image. Three types: 1) Image format; tiff, png, jpg, etc. 2) external 3) raw.', 'default': None}, {'name': 'external_file', 'type': ('array_data', 'data'), 'doc': 'Path or URL to one or more external file(s). Field only present if format=external. ' - 'Either external_file or data must be specified, but not both.', 'default': None}, + 'Either external_file or data must be specified (not None), but not both.', 'default': None}, {'name': 'starting_frame', 'type': Iterable, 'doc': 'Each entry is the frame number in the corresponding external_file variable. ' 'This serves as an index to what frames each file contains. If external_file is not ' @@ -48,10 +58,22 @@ class ImageSeries(TimeSeries): def __init__(self, **kwargs): bits_per_pixel, dimension, external_file, starting_frame, format, device = popargs( 'bits_per_pixel', 'dimension', 'external_file', 'starting_frame', 'format', 'device', kwargs) - call_docval_func(super(ImageSeries, self).__init__, kwargs) - if external_file is None and self.data is None: + name, data, unit = getargs('name', 'data', 'unit', kwargs) + if data is not None and unit is None: + raise ValueError("Must supply 'unit' argument when supplying 'data' to %s '%s'." + % (self.__class__.__name__, name)) + if external_file is None and data is None: raise ValueError("Must supply either external_file or data to %s '%s'." - % (self.__class__.__name__, self.name)) + % (self.__class__.__name__, name)) + + # data and unit are required in TimeSeries, but allowed to be None here, so handle this specially + if data is None: + kwargs['data'] = ImageSeries.DEFAULT_DATA + if unit is None: + kwargs['unit'] = ImageSeries.DEFAULT_UNIT + + call_docval_func(super(ImageSeries, self).__init__, kwargs) + self.bits_per_pixel = bits_per_pixel self.dimension = dimension self.external_file = external_file @@ -76,7 +98,7 @@ def bits_per_pixel(self, val): @register_class('IndexSeries', CORE_NAMESPACE) class IndexSeries(TimeSeries): ''' - Stores indices to image frames stored in an ImageSeries. The purpose of the ImageIndexSeries is to allow + Stores indices to image frames stored in an ImageSeries. The purpose of the IndexSeries is to allow a static image stack to be stored somewhere, and the images in the stack to be referenced out-of-order. This can be for the display of individual images, or of movie segments (as a movie is simply a series of images). The data field stores the index of the frame in the referenced ImageSeries, and the timestamps @@ -85,10 +107,13 @@ class IndexSeries(TimeSeries): __nwbfields__ = ('indexed_timeseries',) + # # value used when an ImageSeries is read and missing data + # DEFAULT_UNIT = 'N/A' + @docval(*get_docval(TimeSeries.__init__, 'name'), # required {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': (None, ), # required - 'doc': ('The data values. Must be 1D, where the first dimension must be time (frame)')}, - *get_docval(TimeSeries.__init__, 'unit'), + 'doc': ('The data values. Must be 1D, where the first dimension must be time (frame)')}, + *get_docval(TimeSeries.__init__, 'unit'), # required {'name': 'indexed_timeseries', 'type': TimeSeries, # required 'doc': 'HDF5 link to TimeSeries containing images that are indexed.'}, *get_docval(TimeSeries.__init__, 'resolution', 'conversion', 'timestamps', 'starting_time', 'rate', @@ -111,12 +136,11 @@ class ImageMaskSeries(ImageSeries): __nwbfields__ = ('masked_imageseries',) @docval(*get_docval(ImageSeries.__init__, 'name'), # required - *get_docval(ImageSeries.__init__, 'data', 'unit'), {'name': 'masked_imageseries', 'type': ImageSeries, # required 'doc': 'Link to ImageSeries that mask is applied to.'}, - *get_docval(ImageSeries.__init__, 'format', 'external_file', 'starting_frame', 'bits_per_pixel', - 'dimension', 'resolution', 'conversion', 'timestamps', 'starting_time', 'rate', 'comments', - 'description', 'control', 'control_description'), + *get_docval(ImageSeries.__init__, 'data', 'unit', 'format', 'external_file', 'starting_frame', + 'bits_per_pixel', 'dimension', 'resolution', 'conversion', 'timestamps', 'starting_time', + 'rate', 'comments', 'description', 'control', 'control_description'), {'name': 'device', 'type': Device, 'doc': ('Device used to capture the mask data. This field will likely not be needed. ' 'The device used to capture the masked ImageSeries data should be stored in the ImageSeries.'), @@ -141,19 +165,20 @@ class OpticalSeries(ImageSeries): 'field_of_view', 'orientation') - @docval(*get_docval(ImageSeries.__init__, 'name'), - {'name': 'data', 'type': ('array_data', 'data'), 'shape': ([None] * 3, [None, None, None, 3]), - 'doc': ('Images presented to subject, either grayscale or RGB. May be 3D or 4D. The first dimension must ' - 'be time (frame). The second and third dimensions represent x and y. The optional fourth ' - 'dimension must be length 3 and represents the RGB value for color images.')}, - *get_docval(ImageSeries.__init__, 'unit', 'format'), + @docval(*get_docval(ImageSeries.__init__, 'name'), # required {'name': 'distance', 'type': 'float', 'doc': 'Distance from camera/monitor to target/eye.'}, # required {'name': 'field_of_view', 'type': ('array_data', 'data', 'TimeSeries'), 'shape': ((2, ), (3, )), # required 'doc': 'Width, height and depth of image, or imaged area (meters).'}, {'name': 'orientation', 'type': str, # required 'doc': 'Description of image relative to some reference frame (e.g., which way is up). ' 'Must also specify frame of reference.'}, - *get_docval(ImageSeries.__init__, 'external_file', 'starting_frame', 'bits_per_pixel', + {'name': 'data', 'type': ('array_data', 'data'), 'shape': ([None] * 3, [None, None, None, 3]), + 'doc': ('Images presented to subject, either grayscale or RGB. May be 3D or 4D. The first dimension must ' + 'be time (frame). The second and third dimensions represent x and y. The optional fourth ' + 'dimension must be length 3 and represents the RGB value for color images. Either data or ' + 'external_file must be specified, but not both.'), + 'default': None}, + *get_docval(ImageSeries.__init__, 'unit', 'format', 'external_file', 'starting_frame', 'bits_per_pixel', 'dimension', 'resolution', 'conversion', 'timestamps', 'starting_time', 'rate', 'comments', 'description', 'control', 'control_description', 'device')) def __init__(self, **kwargs): diff --git a/src/pynwb/io/base.py b/src/pynwb/io/base.py index b1aaa940b..7067ddd06 100644 --- a/src/pynwb/io/base.py +++ b/src/pynwb/io/base.py @@ -72,10 +72,47 @@ def timestamps_carg(self, builder, manager): # # NOTE: it is not available when data is externally linked # and we haven't explicitly read that file - if tstamps_builder.builder.parent is not None: - target = tstamps_builder.builder + target = tstamps_builder.builder + if target.parent is not None: return manager.construct(target.parent) else: - return tstamps_builder.builder.data + return target.data else: return tstamps_builder.data + + @NWBContainerMapper.constructor_arg("data") + def data_carg(self, builder, manager): + # handle case where a TimeSeries is read and missing data + timeseries_cls = manager.get_cls(builder) + data_builder = builder.get('data') + if data_builder is None: + return timeseries_cls.DEFAULT_DATA + if isinstance(data_builder, LinkBuilder): + # NOTE: parent is not available when data is externally linked + # and we haven't explicitly read that file + target = data_builder.builder + if target.parent is not None: + return manager.construct(target.parent) + else: + return target.data + return data_builder.data + + @NWBContainerMapper.constructor_arg("unit") + def unit_carg(self, builder, manager): + # handle case where a TimeSeries is read and missing unit + timeseries_cls = manager.get_cls(builder) + data_builder = builder.get('data') + if data_builder is None: + return timeseries_cls.DEFAULT_UNIT + if isinstance(data_builder, LinkBuilder): + # NOTE: parent is not available when data is externally linked + # and we haven't explicitly read that file + target = data_builder.builder + if target.parent is not None: + data_builder = manager.construct(target.parent) + else: + data_builder = target + unit_value = data_builder.attributes.get('unit') + if unit_value is None: + return timeseries_cls.DEFAULT_UNIT + return unit_value diff --git a/src/pynwb/io/image.py b/src/pynwb/io/image.py index bb6b0212e..afa47c35c 100644 --- a/src/pynwb/io/image.py +++ b/src/pynwb/io/image.py @@ -1,5 +1,4 @@ from .. import register_map - from ..image import ImageSeries from .base import TimeSeriesMap @@ -8,6 +7,6 @@ class ImageSeriesMap(TimeSeriesMap): def __init__(self, spec): - super(ImageSeriesMap, self).__init__(spec) + super().__init__(spec) external_file_spec = self.spec.get_dataset('external_file') self.map_spec('starting_frame', external_file_spec.get_attribute('starting_frame')) diff --git a/src/pynwb/misc.py b/src/pynwb/misc.py index 31303edf1..0e43dfd52 100644 --- a/src/pynwb/misc.py +++ b/src/pynwb/misc.py @@ -12,19 +12,16 @@ @register_class('AnnotationSeries', CORE_NAMESPACE) class AnnotationSeries(TimeSeries): - """ - Stores text-based records about the experiment. To use the - AnnotationSeries, add records individually through - add_annotation() and then call finalize(). Alternatively, if - all annotations are already stored in a list, use set_data() - and set_timestamps() + """Stores text-based records about the experiment. + To use the AnnotationSeries, add records individually through add_annotation(). Alternatively, if all annotations + are already stored in a list or numpy array, set the data and timestamps in the constructor. """ __nwbfields__ = () @docval(*get_docval(TimeSeries.__init__, 'name'), # required {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': (None,), - 'doc': 'The data values over time. Must be 1D.', + 'doc': 'The annotations over time. Must be 1D.', 'default': list()}, *get_docval(TimeSeries.__init__, 'timestamps', 'comments', 'description')) def __init__(self, **kwargs): @@ -34,9 +31,7 @@ def __init__(self, **kwargs): @docval({'name': 'time', 'type': 'float', 'doc': 'The time for the annotation'}, {'name': 'annotation', 'type': str, 'doc': 'the annotation'}) def add_annotation(self, **kwargs): - ''' - Add an annotation - ''' + """Add an annotation.""" time, annotation = getargs('time', 'annotation', kwargs) self.fields['timestamps'].append(time) self.fields['data'].append(annotation) @@ -255,6 +250,9 @@ class DecompositionSeries(TimeSeries): {'name': 'bands', 'doc': 'the bands that the signal is decomposed into', 'child': True}) + # value used when a DecompositionSeries is read and missing data + DEFAULT_DATA = np.ndarray(shape=(0, 0, 0), dtype=np.uint8) + @docval(*get_docval(TimeSeries.__init__, 'name'), # required {'name': 'data', 'type': ('array_data', 'data', TimeSeries), # required 'doc': ('The data values. Must be 3D, where the first dimension must be time, the second dimension must ' diff --git a/src/pynwb/nwb-schema b/src/pynwb/nwb-schema index 55f264d19..884b90a22 160000 --- a/src/pynwb/nwb-schema +++ b/src/pynwb/nwb-schema @@ -1 +1 @@ -Subproject commit 55f264d19f11aa2bdbbeef98eab3ace3e2e9fcf8 +Subproject commit 884b90a22135ca4f111d727f7fee1a8a52b58633 diff --git a/src/pynwb/testing/make_test_files.py b/src/pynwb/testing/make_test_files.py index b61af37b5..6dcbf148d 100644 --- a/src/pynwb/testing/make_test_files.py +++ b/src/pynwb/testing/make_test_files.py @@ -1,5 +1,8 @@ -from pynwb import NWBFile, NWBHDF5IO, validate, __version__ +import numpy as np + from datetime import datetime +from pynwb import NWBFile, NWBHDF5IO, __version__, TimeSeries +from pynwb.image import ImageSeries # pynwb 1.0.2 should be installed with hdmf 1.0.3 # pynwb 1.0.3 should be installed with hdmf 1.0.5 @@ -13,12 +16,10 @@ def _write(test_name, nwbfile): with NWBHDF5IO(filename, 'w') as io: io.write(nwbfile) - with NWBHDF5IO(filename, 'r') as io: - validate(io) - nwbfile = io.read() + return filename -def make_nwbfile(): +def make_nwbfile_empty(): nwbfile = NWBFile(session_description='ADDME', identifier='ADDME', session_start_time=datetime.now().astimezone()) @@ -44,7 +45,83 @@ def make_nwbfile_str_pub(): _write(test_name, nwbfile) +def make_nwbfile_timeseries_no_data(): + nwbfile = NWBFile(session_description='ADDME', + identifier='ADDME', + session_start_time=datetime.now().astimezone()) + ts = TimeSeries( + name='test_timeseries', + rate=1., + unit='unit', + ) + nwbfile.add_acquisition(ts) + + test_name = 'timeseries_no_data' + _write(test_name, nwbfile) + + +def make_nwbfile_timeseries_no_unit(): + nwbfile = NWBFile(session_description='ADDME', + identifier='ADDME', + session_start_time=datetime.now().astimezone()) + ts = TimeSeries( + name='test_timeseries', + data=[0], + rate=1., + ) + nwbfile.add_acquisition(ts) + + test_name = 'timeseries_no_unit' + _write(test_name, nwbfile) + + +def make_nwbfile_imageseries_no_data(): + nwbfile = NWBFile(session_description='ADDME', + identifier='ADDME', + session_start_time=datetime.now().astimezone()) + image_series = ImageSeries( + name='test_imageseries', + external_file=['external_file'], + starting_frame=[1, 2, 3], + format='tiff', + timestamps=[1., 2., 3.] + ) + + nwbfile.add_acquisition(image_series) + + test_name = 'imageseries_no_data' + _write(test_name, nwbfile) + + +def make_nwbfile_imageseries_no_unit(): + """Create a test file with an ImageSeries with data and no unit.""" + nwbfile = NWBFile(session_description='ADDME', + identifier='ADDME', + session_start_time=datetime.now().astimezone()) + image_series = ImageSeries( + name='test_imageseries', + data=np.ones((3, 3, 3)), + external_file=['external_file'], + starting_frame=[1, 2, 3], + format='tiff', + timestamps=[1., 2., 3.] + ) + + nwbfile.add_acquisition(image_series) + + test_name = 'imageseries_no_unit' + _write(test_name, nwbfile) + + if __name__ == '__main__': - make_nwbfile() - make_nwbfile_str_experimenter() - make_nwbfile_str_pub() + + if __version__ == '1.1.2': + make_nwbfile_empty() + make_nwbfile_str_experimenter() + make_nwbfile_str_pub() + + if __version__ == '1.5.1': + make_nwbfile_timeseries_no_data() + make_nwbfile_timeseries_no_unit() + make_nwbfile_imageseries_no_data() + make_nwbfile_imageseries_no_unit() diff --git a/src/pynwb/testing/testh5io.py b/src/pynwb/testing/testh5io.py index eecc2da27..3627e3ad0 100644 --- a/src/pynwb/testing/testh5io.py +++ b/src/pynwb/testing/testh5io.py @@ -58,9 +58,7 @@ def setUpContainer(self): raise NotImplementedError('Cannot run test unless setUpContainer is implemented') def test_roundtrip(self): - """ - Test whether the test Container read from file has the same contents as the original test Container and - validate the file + """Test whether the read Container has the same contents as the original Container and validate the file. """ self.read_container = self.roundtripContainer() self.assertIsNotNone(str(self.container)) # added as a test to make sure printing works @@ -85,9 +83,7 @@ def test_roundtrip_export(self): self.assertContainerEqual(self.read_container, self.container, ignore_hdmf_attrs=True) def roundtripContainer(self, cache_spec=False): - """ - Add the test Container to an NWBFile, write it to file, read the file, and return the test Container from the - file + """Add the Container to an NWBFile, write it to file, read the file, and return the Container from the file. """ description = 'a file to test writing and reading a %s' % self.container_type identifier = 'TEST_%s' % self.container_type diff --git a/tests/back_compat/1.5.1_imageseries_no_data.nwb b/tests/back_compat/1.5.1_imageseries_no_data.nwb new file mode 100644 index 000000000..b7ca1dceb Binary files /dev/null and b/tests/back_compat/1.5.1_imageseries_no_data.nwb differ diff --git a/tests/back_compat/1.5.1_imageseries_no_unit.nwb b/tests/back_compat/1.5.1_imageseries_no_unit.nwb new file mode 100644 index 000000000..b2784c2c6 Binary files /dev/null and b/tests/back_compat/1.5.1_imageseries_no_unit.nwb differ diff --git a/tests/back_compat/1.5.1_timeseries_no_data.nwb b/tests/back_compat/1.5.1_timeseries_no_data.nwb new file mode 100644 index 000000000..f5a8bbe25 Binary files /dev/null and b/tests/back_compat/1.5.1_timeseries_no_data.nwb differ diff --git a/tests/back_compat/1.5.1_timeseries_no_unit.nwb b/tests/back_compat/1.5.1_timeseries_no_unit.nwb new file mode 100644 index 000000000..dbf17442f Binary files /dev/null and b/tests/back_compat/1.5.1_timeseries_no_unit.nwb differ diff --git a/tests/back_compat/test_read.py b/tests/back_compat/test_read.py index c9ccd51ce..d3a813ead 100644 --- a/tests/back_compat/test_read.py +++ b/tests/back_compat/test_read.py @@ -1,7 +1,9 @@ +import numpy as np from pathlib import Path import warnings -from pynwb import NWBHDF5IO, validate +from pynwb import NWBHDF5IO, validate, TimeSeries +from pynwb.image import ImageSeries from pynwb.testing import TestCase @@ -19,9 +21,10 @@ class TestReadOldVersions(TestCase): } def test_read(self): - """ - Attempt to read and validate all NWB files in the same folder as this file. The folder should contain NWB files - from previous versions of NWB. See src/pynwb/testing/make_test_files.py for code to generate the NWB files. + """Test reading and validating all NWB files in the same folder as this file. + + This folder contains NWB files generated by previous versions of NWB using the script + src/pynwb/testing/make_test_files.py """ dir_path = Path(__file__).parent nwb_files = dir_path.glob('*.nwb') @@ -36,3 +39,31 @@ def test_read(self): warnings.warn('%s: %s' % (f.name, e)) # TODO uncomment below when validation errors have been fixed # raise Exception('%d validation error(s). See warnings.' % len(errors)) + + def test_read_timeseries_no_data(self): + """Test that a TimeSeries written without data is read with data set to the default value.""" + f = Path(__file__).parent / '1.5.1_timeseries_no_data.nwb' + with NWBHDF5IO(str(f), 'r') as io: + read_nwbfile = io.read() + np.testing.assert_array_equal(read_nwbfile.acquisition['test_timeseries'].data, TimeSeries.DEFAULT_DATA) + + def test_read_timeseries_no_unit(self): + """Test that an ImageSeries written without unit is read with unit set to the default value.""" + f = Path(__file__).parent / '1.5.1_timeseries_no_unit.nwb' + with NWBHDF5IO(str(f), 'r') as io: + read_nwbfile = io.read() + self.assertEqual(read_nwbfile.acquisition['test_timeseries'].unit, TimeSeries.DEFAULT_UNIT) + + def test_read_imageseries_no_data(self): + """Test that an ImageSeries written without data is read with data set to the default value.""" + f = Path(__file__).parent / '1.5.1_imageseries_no_data.nwb' + with NWBHDF5IO(str(f), 'r') as io: + read_nwbfile = io.read() + np.testing.assert_array_equal(read_nwbfile.acquisition['test_imageseries'].data, ImageSeries.DEFAULT_DATA) + + def test_read_imageseries_no_unit(self): + """Test that an ImageSeries written without unit is read with unit set to the default value.""" + f = Path(__file__).parent / '1.5.1_imageseries_no_unit.nwb' + with NWBHDF5IO(str(f), 'r') as io: + read_nwbfile = io.read() + self.assertEqual(read_nwbfile.acquisition['test_imageseries'].unit, ImageSeries.DEFAULT_UNIT) diff --git a/tests/integration/hdf5/test_ophys.py b/tests/integration/hdf5/test_ophys.py index 6507efccf..e882cfaf8 100644 --- a/tests/integration/hdf5/test_ophys.py +++ b/tests/integration/hdf5/test_ophys.py @@ -175,9 +175,14 @@ def buildPlaneSegmentation(self): (7, 8, 2.0), (9, 10, 2.)] ts = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] - self.image_series = ImageSeries(name='test_iS', dimension=[2], - external_file=['images.tiff'], - starting_frame=[1, 2, 3], format='tiff', timestamps=ts) + self.image_series = ImageSeries( + name='test_iS', + dimension=[2], + external_file=['images.tiff'], + starting_frame=[1, 2, 3], + format='tiff', + timestamps=ts + ) self.device = Device(name='dev1') self.optical_channel = OpticalChannel( @@ -239,9 +244,14 @@ class MaskIO(TestPlaneSegmentationIO, metaclass=ABCMeta): def buildPlaneSegmentationNoRois(self): """ Return an PlaneSegmentation and set related objects """ ts = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] - self.image_series = ImageSeries(name='test_iS', dimension=[2], - external_file=['images.tiff'], - starting_frame=[1, 2, 3], format='tiff', timestamps=ts) + self.image_series = ImageSeries( + name='test_iS', + dimension=[2], + external_file=['images.tiff'], + starting_frame=[1, 2, 3], + format='tiff', + timestamps=ts + ) self.device = Device(name='dev1') self.optical_channel = OpticalChannel( name='test_optical_channel', diff --git a/tests/unit/test_base.py b/tests/unit/test_base.py index b2579e8f1..dc0869f0d 100644 --- a/tests/unit/test_base.py +++ b/tests/unit/test_base.py @@ -136,11 +136,6 @@ def test_bad_continuity_timeseries(self): 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5], continuity='wrong') - def test_nodata(self): - ts1 = TimeSeries('test_ts1', starting_time=0.0, rate=0.1) - with self.assertWarns(UserWarning): - self.assertIs(ts1.num_samples, None) - def test_dataio_list_data(self): length = 100 data = list(range(length)) @@ -197,7 +192,7 @@ def test_no_time(self): def test_no_starting_time(self): # if no starting_time is given, 0.0 is assumed - ts1 = TimeSeries('test_ts1', rate=0.1) + ts1 = TimeSeries('test_ts1', data=[1, 2, 3], unit='unit', rate=0.1) self.assertEqual(ts1.starting_time, 0.0) def test_conflicting_time_args(self): @@ -248,7 +243,7 @@ def test_get_length1_valid_data(self): """Get data from a TimeSeriesReferenceVectorData with one element and valid data""" temp = TimeSeriesReferenceVectorData() value = TimeSeriesReference(0, 5, TimeSeries(name='test', description='test', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) temp.append(value) self.assertTupleEqual(temp[0], value) self.assertListEqual(temp[:], [TimeSeriesReferenceVectorData.TIME_SERIES_REFERENCE_TUPLE(*value), ]) @@ -257,7 +252,7 @@ def test_get_length1_invalid_data(self): """Get data from a TimeSeriesReferenceVectorData with one element and invalid data""" temp = TimeSeriesReferenceVectorData() value = TimeSeriesReference(-1, -1, TimeSeries(name='test', description='test', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) temp.append(value) # test index slicing re = temp[0] @@ -277,7 +272,7 @@ def test_get_length5_valid_data(self): temp = TimeSeriesReferenceVectorData() num_values = 5 values = [TimeSeriesReference(0, 5, TimeSeries(name='test'+str(i), description='test', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) for i in range(num_values)] for v in values: temp.append(v) @@ -299,13 +294,14 @@ def test_get_length5_with_invalid_data(self): temp = TimeSeriesReferenceVectorData() num_values = 5 values = [TimeSeriesReference(0, 5, TimeSeries(name='test'+str(i+1), description='test', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) for i in range(num_values-2)] values = ([TimeSeriesReference(-1, -1, TimeSeries(name='test'+str(0), description='test', - data=np.arange(10), starting_time=5.0, rate=0.1)), ] + data=np.arange(10), unit='unit', starting_time=5.0, + rate=0.1)), ] + values + [TimeSeriesReference(-1, -1, TimeSeries(name='test'+str(5), description='test', - data=np.arange(10), starting_time=5.0, rate=0.1)), ]) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)), ]) for v in values: temp.append(v) # Test single element selection @@ -347,21 +343,21 @@ class TestTimeSeriesReference(TestCase): def test_check_types(self): # invalid selection but with correct types tsr = TimeSeriesReference(-1, -1, TimeSeries(name='test'+str(0), description='test', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) self.assertTrue(tsr.check_types()) # invalid types, use float instead of int for both idx_start and count tsr = TimeSeriesReference(1.0, 5.0, TimeSeries(name='test'+str(0), description='test', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) with self.assertRaisesWith(TypeError, "idx_start must be an integer not "): tsr.check_types() # invalid types, use float instead of int for idx_start only tsr = TimeSeriesReference(1.0, 5, TimeSeries(name='test'+str(0), description='test', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) with self.assertRaisesWith(TypeError, "idx_start must be an integer not "): tsr.check_types() # invalid types, use float instead of int for count only tsr = TimeSeriesReference(1, 5.0, TimeSeries(name='test'+str(0), description='test', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) with self.assertRaisesWith(TypeError, "count must be an integer "): tsr.check_types() # invalid type for TimeSeries but valid idx_start and count @@ -371,74 +367,76 @@ def test_check_types(self): def test_is_invalid(self): tsr = TimeSeriesReference(-1, -1, TimeSeries(name='test'+str(0), description='test', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) self.assertFalse(tsr.isvalid()) def test_is_valid(self): tsr = TimeSeriesReference(0, 10, TimeSeries(name='test'+str(0), description='test', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) self.assertTrue(tsr.isvalid()) def test_is_valid_bad_index(self): # Error: negative start_index but positive count tsr = TimeSeriesReference(-1, 10, TimeSeries(name='test0', description='test0', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) with self.assertRaisesWith(IndexError, "'idx_start' -1 out of range for timeseries 'test0'"): tsr.isvalid() # Error: start_index too large tsr = TimeSeriesReference(10, 0, TimeSeries(name='test0', description='test0', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) with self.assertRaisesWith(IndexError, "'idx_start' 10 out of range for timeseries 'test0'"): tsr.isvalid() # Error: positive start_index but negative count tsr = TimeSeriesReference(0, -3, TimeSeries(name='test0', description='test0', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) with self.assertRaisesWith(IndexError, "'count' -3 invalid. 'count' must be positive"): tsr.isvalid() # Error: start_index + count too large tsr = TimeSeriesReference(3, 10, TimeSeries(name='test0', description='test0', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) with self.assertRaisesWith(IndexError, "'idx_start + count' out of range for timeseries 'test0'"): tsr.isvalid() def test_timestamps_property(self): # Timestamps from starting_time and rate tsr = TimeSeriesReference(5, 4, TimeSeries(name='test0', description='test0', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) np.testing.assert_array_equal(tsr.timestamps, np.array([5.5, 5.6, 5.7, 5.8])) # Timestamps from timestamps directly tsr = TimeSeriesReference(5, 4, TimeSeries(name='test0', description='test0', - data=np.arange(10), timestamps=np.arange(10).astype(float))) + data=np.arange(10), unit='unit', + timestamps=np.arange(10).astype(float))) np.testing.assert_array_equal(tsr.timestamps, np.array([5., 6., 7., 8.])) def test_timestamps_property_invalid_reference(self): # Timestamps from starting_time and rate tsr = TimeSeriesReference(-1, -1, TimeSeries(name='test0', description='test0', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) self.assertIsNone(tsr.timestamps) def test_timestamps_property_bad_reference(self): tsr = TimeSeriesReference(0, 12, TimeSeries(name='test0', description='test0', - data=np.arange(10), timestamps=np.arange(10).astype(float))) + data=np.arange(10), unit='unit', + timestamps=np.arange(10).astype(float))) with self.assertRaisesWith(IndexError, "'idx_start + count' out of range for timeseries 'test0'"): tsr.timestamps tsr = TimeSeriesReference(0, 12, TimeSeries(name='test0', description='test0', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) with self.assertRaisesWith(IndexError, "'idx_start + count' out of range for timeseries 'test0'"): tsr.timestamps def test_data_property(self): tsr = TimeSeriesReference(5, 4, TimeSeries(name='test0', description='test0', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) np.testing.assert_array_equal(tsr.data, np.array([5., 6., 7., 8.])) def test_data_property_invalid_reference(self): tsr = TimeSeriesReference(-1, -1, TimeSeries(name='test0', description='test0', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) self.assertIsNone(tsr.data) def test_data_property_bad_reference(self): tsr = TimeSeriesReference(0, 12, TimeSeries(name='test0', description='test0', - data=np.arange(10), starting_time=5.0, rate=0.1)) + data=np.arange(10), unit='unit', starting_time=5.0, rate=0.1)) with self.assertRaisesWith(IndexError, "'idx_start + count' out of range for timeseries 'test0'"): tsr.data diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index ee599b00e..ef95a058e 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -41,9 +41,19 @@ class TestPrint(TestCase): def test_print_file(self): nwbfile = NWBFile(session_description='session_description', identifier='identifier', session_start_time=datetime.now(tzlocal())) - ts = TimeSeries('name', [1., 2., 3.] * 1000, timestamps=[1, 2, 3]) - ts2 = TimeSeries('name2', [1, 2, 3] * 1000, timestamps=[1, 2, 3]) - expected = """name pynwb.base.TimeSeries at 0x%d + ts1 = TimeSeries( + name='name1', + data=[1., 2., 3.] * 1000, + unit='unit', + timestamps=[1, 2, 3] + ) + ts2 = TimeSeries( + name='name2', + data=[1, 2, 3] * 1000, + unit='unit', + timestamps=[1, 2, 3] + ) + expected = """name1 pynwb.base.TimeSeries at 0x%d Fields: comments: no comments conversion: 1.0 @@ -53,16 +63,17 @@ def test_print_file(self): resolution: -1.0 timestamps: [1 2 3] timestamps_unit: seconds + unit: unit """ - expected %= id(ts) - self.assertEqual(str(ts), expected) - nwbfile.add_acquisition(ts) + expected %= id(ts1) + self.assertEqual(str(ts1), expected) + nwbfile.add_acquisition(ts1) nwbfile.add_acquisition(ts2) nwbfile.add_epoch(start_time=1.0, stop_time=10.0, tags=['tag1', 'tag2']) expected_re = r"""root pynwb\.file\.NWBFile at 0x\d+ Fields: acquisition: { - name , + name1 , name2 } epoch_tags: { diff --git a/tests/unit/test_epoch.py b/tests/unit/test_epoch.py index 64278130d..b525356d1 100644 --- a/tests/unit/test_epoch.py +++ b/tests/unit/test_epoch.py @@ -12,8 +12,8 @@ class TimeIntervalsTest(TestCase): def test_init(self): tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float64) - ts = TimeSeries("test_ts", list(range(len(tstamps))), 'unit', timestamps=tstamps) - ept = TimeIntervals('epochs', "TimeIntervals unittest") + ts = TimeSeries(name="test_ts", data=list(range(len(tstamps))), unit='unit', timestamps=tstamps) + ept = TimeIntervals(name='epochs', description="TimeIntervals unittest") self.assertEqual(ept.name, 'epochs') ept.add_interval(10.0, 20.0, ["test", "unittest", "pynwb"], ts) row = ept[0] @@ -25,8 +25,8 @@ def test_init(self): def get_timeseries(self): return [ - TimeSeries(name='a', timestamps=np.linspace(0, 1, 11)), - TimeSeries(name='b', timestamps=np.linspace(0.1, 5, 13)), + TimeSeries(name='a', data=[1]*11, unit='unit', timestamps=np.linspace(0, 1, 11)), + TimeSeries(name='b', data=[1]*13, unit='unit', timestamps=np.linspace(0.1, 5, 13)), ] def get_dataframe(self): diff --git a/tests/unit/test_image.py b/tests/unit/test_image.py index c79a7b0d5..ef6f50cf7 100644 --- a/tests/unit/test_image.py +++ b/tests/unit/test_image.py @@ -55,15 +55,43 @@ def test_data_no_frame(self): ) self.assertIsNone(iS.starting_frame) + def test_data_no_unit(self): + msg = "Must supply 'unit' argument when supplying 'data' to ImageSeries 'test_iS'." + with self.assertRaisesWith(ValueError, msg): + ImageSeries( + name='test_iS', + data=np.ones((3, 3, 3)), + timestamps=list() + ) + + def test_external_file_no_unit(self): + iS = ImageSeries( + name='test_iS', + external_file=['external_file'], + timestamps=list() + ) + self.assertEqual(iS.unit, ImageSeries.DEFAULT_UNIT) + class IndexSeriesConstructor(TestCase): def test_init(self): - ts = TimeSeries('test_ts', list(), 'unit', timestamps=list()) - iS = IndexSeries('test_iS', list(), ts, unit='unit', timestamps=list()) + ts = TimeSeries( + name='test_ts', + data=[1, 2, 3], + unit='unit', + timestamps=[0.1, 0.2, 0.3] + ) + iS = IndexSeries( + name='test_iS', + data=[1, 2, 3], + unit='N/A', + indexed_timeseries=ts, + timestamps=[0.1, 0.2, 0.3] + ) self.assertEqual(iS.name, 'test_iS') - self.assertEqual(iS.unit, 'unit') - self.assertEqual(iS.indexed_timeseries, ts) + self.assertEqual(iS.unit, 'N/A') + self.assertIs(iS.indexed_timeseries, ts) class ImageMaskSeriesConstructor(TestCase): @@ -78,7 +106,7 @@ def test_init(self): format='tiff', timestamps=[1., 2.]) self.assertEqual(ims.name, 'test_ims') self.assertEqual(ims.unit, 'unit') - self.assertEqual(ims.masked_imageseries, iS) + self.assertIs(ims.masked_imageseries, iS) self.assertEqual(ims.external_file, ['external_file']) self.assertEqual(ims.starting_frame, [1, 2, 3]) self.assertEqual(ims.format, 'tiff') diff --git a/tests/unit/test_misc.py b/tests/unit/test_misc.py index 4412063ce..c8170d9fe 100644 --- a/tests/unit/test_misc.py +++ b/tests/unit/test_misc.py @@ -11,7 +11,7 @@ class AnnotationSeriesConstructor(TestCase): def test_init(self): - aS = AnnotationSeries('test_aS', timestamps=list()) + aS = AnnotationSeries('test_aS', data=[1, 2, 3], timestamps=list()) self.assertEqual(aS.name, 'test_aS') aS.add_annotation(2.0, 'comment') diff --git a/tests/unit/test_ogen.py b/tests/unit/test_ogen.py index 82f873de9..678f284dc 100644 --- a/tests/unit/test_ogen.py +++ b/tests/unit/test_ogen.py @@ -7,14 +7,25 @@ class OptogeneticSeriesConstructor(TestCase): def test_init(self): device = Device('name') - oS = OptogeneticStimulusSite('site1', device, 'description', 300., 'location') + oS = OptogeneticStimulusSite( + name='site1', + device=device, + description='description', + excitation_lambda=300., + location='location' + ) self.assertEqual(oS.name, 'site1') self.assertEqual(oS.device, device) self.assertEqual(oS.description, 'description') self.assertEqual(oS.excitation_lambda, 300.) self.assertEqual(oS.location, 'location') - iS = OptogeneticSeries('test_iS', list(), oS, timestamps=list()) + iS = OptogeneticSeries( + name='test_iS', + data=[1, 2, 3], + site=oS, + timestamps=[0.1, 0.2, 0.3] + ) self.assertEqual(iS.name, 'test_iS') self.assertEqual(iS.unit, 'watts') self.assertEqual(iS.site, oS) diff --git a/tests/unit/test_ophys.py b/tests/unit/test_ophys.py index 024c081cb..87e40a421 100644 --- a/tests/unit/test_ophys.py +++ b/tests/unit/test_ophys.py @@ -280,10 +280,10 @@ def test_init(self): ts = RoiResponseSeries( name='test_ts', - data=list(), + data=[1, 2, 3], rois=rt_region, unit='unit', - timestamps=list() + timestamps=[0.1, 0.2, 0.3] ) self.assertEqual(ts.name, 'test_ts') self.assertEqual(ts.unit, 'unit') @@ -297,10 +297,10 @@ def test_init(self): rrs = RoiResponseSeries( name='test_ts', - data=list(), + data=[1, 2, 3], rois=rt_region, unit='unit', - timestamps=list() + timestamps=[0.1, 0.2, 0.3] ) dof = DfOverF(rrs) @@ -314,10 +314,10 @@ def test_init(self): ts = RoiResponseSeries( name='test_ts', - data=list(), + data=[1, 2, 3], rois=rt_region, unit='unit', - timestamps=list() + timestamps=[0.1, 0.2, 0.3] ) ff = Fluorescence(ts)