From 2cbb72519c02ee77c0d2e4cdb14e9e178071b410 Mon Sep 17 00:00:00 2001 From: Andrew Tritt Date: Thu, 14 Mar 2019 16:36:47 -0700 Subject: [PATCH] Add HDMF to PyNWB (#850) * Replace FORM (pynwb.form) with HMDF (hdmf) * add HDMF to requirements.txt * get requirements from requirements.txt in setup.py --- docs/code/creating-and-writing-nwbfile.py | 2 +- docs/gallery/general/advanced_hdf5_io.py | 18 +- docs/gallery/general/extensions.py | 12 +- docs/gallery/general/iterative_write.py | 66 +- docs/gallery/general/linking_data.py | 10 +- requirements-dev.txt | 2 +- requirements.txt | 3 +- setup.py | 18 +- src/pynwb/__init__.py | 18 +- src/pynwb/base.py | 4 +- src/pynwb/behavior.py | 2 +- src/pynwb/core.py | 4 +- src/pynwb/device.py | 2 +- src/pynwb/ecephys.py | 4 +- src/pynwb/epoch.py | 4 +- src/pynwb/file.py | 4 +- src/pynwb/form/__init__.py | 18 - src/pynwb/form/array.py | 197 --- src/pynwb/form/backends/__init__.py | 2 - src/pynwb/form/backends/hdf5/__init__.py | 7 - src/pynwb/form/backends/hdf5/h5_utils.py | 294 --- src/pynwb/form/backends/hdf5/h5tools.py | 938 ---------- src/pynwb/form/backends/io.py | 68 - src/pynwb/form/build/__init__.py | 11 - src/pynwb/form/build/builders.py | 536 ------ src/pynwb/form/build/map.py | 1576 ----------------- src/pynwb/form/build/warnings.py | 15 - src/pynwb/form/container.py | 123 -- src/pynwb/form/data_utils.py | 559 ------ src/pynwb/form/monitor.py | 76 - src/pynwb/form/query.py | 162 -- src/pynwb/form/spec/__init__.py | 20 - src/pynwb/form/spec/catalog.py | 132 -- src/pynwb/form/spec/namespace.py | 432 ----- src/pynwb/form/spec/spec.py | 1216 ------------- src/pynwb/form/spec/write.py | 191 -- src/pynwb/form/utils.py | 583 ------ src/pynwb/form/validate/__init__.py | 5 - src/pynwb/form/validate/errors.py | 144 -- src/pynwb/form/validate/validator.py | 468 ----- src/pynwb/icephys.py | 2 +- src/pynwb/image.py | 2 +- src/pynwb/io/base.py | 2 +- src/pynwb/io/core.py | 8 +- src/pynwb/io/file.py | 2 +- src/pynwb/legacy/__init__.py | 4 +- src/pynwb/legacy/map.py | 4 +- src/pynwb/misc.py | 2 +- src/pynwb/ogen.py | 2 +- src/pynwb/ophys.py | 2 +- src/pynwb/retinotopy.py | 2 +- src/pynwb/spec.py | 4 +- test.py | 8 +- tests/integration/test_io.py | 8 +- tests/integration/ui_write/base.py | 2 +- tests/integration/ui_write/test_base.py | 2 +- tests/integration/ui_write/test_ecephys.py | 2 +- tests/integration/ui_write/test_icephys.py | 2 +- tests/integration/ui_write/test_misc.py | 2 +- tests/integration/ui_write/test_nwbfile.py | 4 +- tests/integration/ui_write/test_ophys.py | 2 +- tests/unit/form_tests/__init__.py | 0 tests/unit/form_tests/build_tests/__init__.py | 0 .../build_tests/test_io_build_builders.py | 313 ---- .../form_tests/build_tests/test_io_manager.py | 318 ---- .../form_tests/build_tests/test_io_map.py | 318 ---- .../build_tests/test_io_map_data.py | 56 - tests/unit/form_tests/spec_tests/__init__.py | 0 .../spec_tests/test_attribute_spec.py | 66 - .../spec_tests/test_dataset_spec.py | 232 --- .../form_tests/spec_tests/test_dtype_spec.py | 62 - .../form_tests/spec_tests/test_group_spec.py | 211 --- .../spec_tests/test_load_namespace.py | 114 -- .../form_tests/spec_tests/test_ref_spec.py | 23 - .../spec_tests/test_spec_catalog.py | 57 - tests/unit/form_tests/test_container.py | 52 - tests/unit/form_tests/test_io_hdf5.py | 274 --- tests/unit/form_tests/test_io_hdf5_h5tools.py | 536 ------ tests/unit/form_tests/test_query.py | 162 -- tests/unit/form_tests/utils_test/__init__.py | 0 tests/unit/form_tests/utils_test/test_core.py | 322 ---- .../utils_test/test_core_DataChunkIterator.py | 129 -- .../utils_test/test_core_ShapeValidator.py | 198 --- .../form_tests/validator_tests/__init__.py | 0 .../validator_tests/test_validate.py | 156 -- tests/unit/pynwb_tests/test_base.py | 4 +- tests/unit/pynwb_tests/test_extension.py | 4 +- tox.ini | 4 +- 88 files changed, 123 insertions(+), 11502 deletions(-) delete mode 100644 src/pynwb/form/__init__.py delete mode 100644 src/pynwb/form/array.py delete mode 100644 src/pynwb/form/backends/__init__.py delete mode 100644 src/pynwb/form/backends/hdf5/__init__.py delete mode 100644 src/pynwb/form/backends/hdf5/h5_utils.py delete mode 100644 src/pynwb/form/backends/hdf5/h5tools.py delete mode 100644 src/pynwb/form/backends/io.py delete mode 100644 src/pynwb/form/build/__init__.py delete mode 100644 src/pynwb/form/build/builders.py delete mode 100644 src/pynwb/form/build/map.py delete mode 100644 src/pynwb/form/build/warnings.py delete mode 100644 src/pynwb/form/container.py delete mode 100644 src/pynwb/form/data_utils.py delete mode 100644 src/pynwb/form/monitor.py delete mode 100644 src/pynwb/form/query.py delete mode 100644 src/pynwb/form/spec/__init__.py delete mode 100644 src/pynwb/form/spec/catalog.py delete mode 100644 src/pynwb/form/spec/namespace.py delete mode 100644 src/pynwb/form/spec/spec.py delete mode 100644 src/pynwb/form/spec/write.py delete mode 100644 src/pynwb/form/utils.py delete mode 100644 src/pynwb/form/validate/__init__.py delete mode 100644 src/pynwb/form/validate/errors.py delete mode 100644 src/pynwb/form/validate/validator.py delete mode 100644 tests/unit/form_tests/__init__.py delete mode 100644 tests/unit/form_tests/build_tests/__init__.py delete mode 100644 tests/unit/form_tests/build_tests/test_io_build_builders.py delete mode 100644 tests/unit/form_tests/build_tests/test_io_manager.py delete mode 100644 tests/unit/form_tests/build_tests/test_io_map.py delete mode 100644 tests/unit/form_tests/build_tests/test_io_map_data.py delete mode 100644 tests/unit/form_tests/spec_tests/__init__.py delete mode 100644 tests/unit/form_tests/spec_tests/test_attribute_spec.py delete mode 100644 tests/unit/form_tests/spec_tests/test_dataset_spec.py delete mode 100644 tests/unit/form_tests/spec_tests/test_dtype_spec.py delete mode 100644 tests/unit/form_tests/spec_tests/test_group_spec.py delete mode 100644 tests/unit/form_tests/spec_tests/test_load_namespace.py delete mode 100644 tests/unit/form_tests/spec_tests/test_ref_spec.py delete mode 100644 tests/unit/form_tests/spec_tests/test_spec_catalog.py delete mode 100644 tests/unit/form_tests/test_container.py delete mode 100644 tests/unit/form_tests/test_io_hdf5.py delete mode 100644 tests/unit/form_tests/test_io_hdf5_h5tools.py delete mode 100644 tests/unit/form_tests/test_query.py delete mode 100644 tests/unit/form_tests/utils_test/__init__.py delete mode 100644 tests/unit/form_tests/utils_test/test_core.py delete mode 100644 tests/unit/form_tests/utils_test/test_core_DataChunkIterator.py delete mode 100644 tests/unit/form_tests/utils_test/test_core_ShapeValidator.py delete mode 100644 tests/unit/form_tests/validator_tests/__init__.py delete mode 100644 tests/unit/form_tests/validator_tests/test_validate.py diff --git a/docs/code/creating-and-writing-nwbfile.py b/docs/code/creating-and-writing-nwbfile.py index 20889c811..98193ed0a 100644 --- a/docs/code/creating-and-writing-nwbfile.py +++ b/docs/code/creating-and-writing-nwbfile.py @@ -136,7 +136,7 @@ def main(): # create-compressed-timeseries: start from pynwb.ecephys import ElectricalSeries from pynwb.behavior import SpatialSeries - from pynwb.form.backends.hdf5 import H5DataIO + from hdmf.backends.hdf5 import H5DataIO ephys_ts = ElectricalSeries('test_compressed_ephys_data', 'an hypothetical source', diff --git a/docs/gallery/general/advanced_hdf5_io.py b/docs/gallery/general/advanced_hdf5_io.py index 8f72e201e..a44d741cd 100644 --- a/docs/gallery/general/advanced_hdf5_io.py +++ b/docs/gallery/general/advanced_hdf5_io.py @@ -8,11 +8,11 @@ ''' #################### -# Wrapping data arrays with :py:meth:`~pynwb.form.backends.hdf5.h5_utils.H5DataIO` +# Wrapping data arrays with :py:meth:`~hdmf.backends.hdf5.h5_utils.H5DataIO` # --------------------------------------------------------------------------------- # # In order to customize the I/O of datasets using the HDF I/O backend we simply need to wrap our datasets -# using :py:meth:`~pynwb.form.backends.hdf5.h5_utils.H5DataIO`. Using H5DataIO allows us to keep the Container +# using :py:meth:`~hdmf.backends.hdf5.h5_utils.H5DataIO`. Using H5DataIO allows us to keep the Container # classes independent of the I/O backend while still allowing us to customize HDF5-specific I/O features. # # Before we get started, lets create an NWBFile for testing so that we can add our data to it. @@ -49,7 +49,7 @@ # Now let's say we want to compress the recorded data values. We now simply need to wrap our data with H5DataIO. # Everything else remains the same -from pynwb.form.backends.hdf5.h5_utils import H5DataIO +from hdmf.backends.hdf5.h5_utils import H5DataIO wrapped_data = H5DataIO(data=data, compression=True) # <---- test_ts = TimeSeries(name='test_compressed_timeseries', data=wrapped_data, # <---- @@ -59,7 +59,7 @@ #################### # This simple approach gives us access to a broad range of advanced I/O features, such as, chunking and -# compression. For a complete list of all available settings see :py:meth:`~pynwb.form.backends.hdf5.h5_utils.H5DataIO` +# compression. For a complete list of all available settings see :py:meth:`~hdmf.backends.hdf5.h5_utils.H5DataIO` #################### # Chunking @@ -81,7 +81,7 @@ #################### -# To use chunking we again, simply need to wrap our dataset via :py:meth:`~pynwb.form.backends.hdf5.h5_utils.H5DataIO`. +# To use chunking we again, simply need to wrap our dataset via :py:meth:`~hdmf.backends.hdf5.h5_utils.H5DataIO`. # Using chunking then also allows to also create resizable arrays simply by defining the ``maxshape`` of the array. data = np.arange(10000).reshape((1000, 10)) @@ -122,7 +122,7 @@ # read/write operations. I/O filters operate on a per-chunk basis in HDF5 and as such require the use of chunking. # Chunking will be automatically enabled by h5py when compression and other I/O filters are enabled. # -# To use compression, we can wrap our dataset using :py:meth:`~pynwb.form.backends.hdf5.h5_utils.H5DataIO` and +# To use compression, we can wrap our dataset using :py:meth:`~hdmf.backends.hdf5.h5_utils.H5DataIO` and # define the approbriate opions: wrapped_data = H5DataIO(data=data, @@ -139,7 +139,7 @@ #################### # .. hint:: # -# In addition to ``compression``, :py:meth:`~pynwb.form.backends.hdf5.h5_utils.H5DataIO` also allows us to +# In addition to ``compression``, :py:meth:`~hdmf.backends.hdf5.h5_utils.H5DataIO` also allows us to # enable the ``shuffle`` and ``fletcher32`` HDF5 I/O filters. #################### @@ -200,10 +200,10 @@ #################### -# Wrapping ``h5py.Datasets`` with :py:meth:`~pynwb.form.backends.hdf5.h5_utils.H5DataIO` +# Wrapping ``h5py.Datasets`` with :py:meth:`~hdmf.backends.hdf5.h5_utils.H5DataIO` # ------------------------------------------------------------------------------------------------ # -# Just for completeness, :py:meth:`~pynwb.form.backends.hdf5.h5_utils.H5DataIO` also allows us to customize +# Just for completeness, :py:meth:`~hdmf.backends.hdf5.h5_utils.H5DataIO` also allows us to customize # how ``h5py.Dataset`` objects should be handled on write by the PyNWBs HDF5 backend via the ``link_data`` # parameter. If ``link_data`` is set to ``True`` then a ``SoftLink`` or ``ExternalLink`` will be created to # point to the HDF5 dataset On the other hand, if ``link_data`` is set to ``False`` then the dataset diff --git a/docs/gallery/general/extensions.py b/docs/gallery/general/extensions.py index 054a647b5..834cc2422 100644 --- a/docs/gallery/general/extensions.py +++ b/docs/gallery/general/extensions.py @@ -28,7 +28,7 @@ # # The following block of code demonstrates how to create a new namespace, and then add a new `neurodata_type` # to this namespace. Finally, -# it calls :py:meth:`~pynwb.form.spec.write.NamespaceBuilder.export` to save the extensions to disk for downstream use. +# it calls :py:meth:`~hdmf.spec.write.NamespaceBuilder.export` to save the extensions to disk for downstream use. from pynwb.spec import NWBNamespaceBuilder, NWBGroupSpec, NWBAttributeSpec @@ -96,7 +96,7 @@ from pynwb import register_class, load_namespaces from pynwb.ecephys import ElectricalSeries -from pynwb.form.utils import docval, call_docval_func, getargs, get_docval +from hdmf.utils import docval, call_docval_func, getargs, get_docval ns_path = "mylab.namespace.yaml" load_namespaces(ns_path) @@ -117,9 +117,9 @@ def __init__(self, **kwargs): #################### # .. note:: # -# See the API docs for more information about :py:func:`~pynwb.form.utils.docval` -# :py:func:`~pynwb.form.utils.call_docval_func`, :py:func:`~pynwb.form.utils.getargs` -# and :py:func:`~pynwb.form.utils.get_docval` +# See the API docs for more information about :py:func:`~hdmf.utils.docval` +# :py:func:`~hdmf.utils.call_docval_func`, :py:func:`~hdmf.utils.getargs` +# and :py:func:`~hdmf.utils.get_docval` # # When extending :py:class:`~pynwb.core.NWBContainer` or :py:class:`~pynwb.core.NWBContainer` # subclasses, you should defining the class field ``__nwbfields__``. This will @@ -151,7 +151,7 @@ def __init__(self, **kwargs): # ----------------------------------------------------- # # Extensions can be cached to file so that your NWB file will carry the extensions needed to read the file with it. -# This is done by setting *cache_spec* to *True* when calling :py:meth:`~pynwb.form.backends.hdf5.h5tools.HDF5IO.write` +# This is done by setting *cache_spec* to *True* when calling :py:meth:`~hdmf.backends.hdf5.h5tools.HDF5IO.write` # on :py:class:`~pynwb.NWBHDF5IO` (See :ref:`basic_writing` for more on writing NWB files). # # To demonstrate this, first we will make some fake data using our extensions. diff --git a/docs/gallery/general/iterative_write.py b/docs/gallery/general/iterative_write.py index cd5bae58e..0388e4aac 100644 --- a/docs/gallery/general/iterative_write.py +++ b/docs/gallery/general/iterative_write.py @@ -53,20 +53,20 @@ # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # In PyNWB the process of iterating over large data arrays is implemented via the concept of -# :py:class:`~pynwb.form.data_utils.DataChunk` and :py:class:`~pynwb.form.data_utils.AbstractDataChunkIterator`. +# :py:class:`~hdmf.data_utils.DataChunk` and :py:class:`~hdmf.data_utils.AbstractDataChunkIterator`. # -# * :py:class:`~pynwb.form.data_utils.DataChunk` is a simple data structure used to describe +# * :py:class:`~hdmf.data_utils.DataChunk` is a simple data structure used to describe # a subset of a larger data array (i.e., a data chunk), consisting of: # # * ``DataChunk.data`` : the array with the data value(s) of the chunk and # * ``DataChunk.selection`` : the NumPy index tuple describing the location of the chunk in the whole array. # -# * :py:class:`~pynwb.form.data_utils.AbstractDataChunkIterator` then defines a class for iterating over large -# data arrays one-:py:class:`~pynwb.form.data_utils.DataChunk`-at-a-time. +# * :py:class:`~hdmf.data_utils.AbstractDataChunkIterator` then defines a class for iterating over large +# data arrays one-:py:class:`~hdmf.data_utils.DataChunk`-at-a-time. # -# * :py:class:`~pynwb.form.data_utils.DataChunkIterator` is a specific implementation of an -# :py:class:`~pynwb.form.data_utils.AbstractDataChunkIterator` that accepts any iterable and assumes -# that we iterate over the first dimension of the data array. :py:class:`~pynwb.form.data_utils.DataChunkIterator` +# * :py:class:`~hdmf.data_utils.DataChunkIterator` is a specific implementation of an +# :py:class:`~hdmf.data_utils.AbstractDataChunkIterator` that accepts any iterable and assumes +# that we iterate over the first dimension of the data array. :py:class:`~hdmf.data_utils.DataChunkIterator` # also supports buffered read, i.e., multiple values from the input iterator can be combined to a single chunk. # This is useful for buffered I/O operations, e.g., to improve performance by accumulating data in memory and # writing larger blocks at once. @@ -77,17 +77,17 @@ # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # On the front end, all a user needs to do is to create or wrap their data in a -# :py:class:`~pynwb.form.data_utils.AbstractDataChunkIterator`. The I/O backend (e.g., -# :py:class:`~pynwb.form.backends.hdf5.h5tools.HDF5IO` or :py:class:`~pynwb.NWBHDF5IO`) then +# :py:class:`~hdmf.data_utils.AbstractDataChunkIterator`. The I/O backend (e.g., +# :py:class:`~hdmf.backends.hdf5.h5tools.HDF5IO` or :py:class:`~pynwb.NWBHDF5IO`) then # implements the iterative processing of the data chunk iterators. PyNWB also provides with -# :py:class:`~pynwb.form.data_utils.DataChunkIterator` a specific implementation of a data chunk iterator +# :py:class:`~hdmf.data_utils.DataChunkIterator` a specific implementation of a data chunk iterator # which we can use to wrap common iterable types (e.g., generators, lists, or numpy arrays). # For more advanced use cases we then need to implement our own derived class of -# :py:class:`~pynwb.form.data_utils.AbstractDataChunkIterator`. +# :py:class:`~hdmf.data_utils.AbstractDataChunkIterator`. # # .. tip:: # -# Currently the HDF5 I/O backend of PyNWB (:py:class:`~pynwb.form.backends.hdf5.h5tools.HDF5IO`, +# Currently the HDF5 I/O backend of PyNWB (:py:class:`~hdmf.backends.hdf5.h5tools.HDF5IO`, # :py:class:`~pynwb.NWBHDF5IO`) processes itertive data writes one-dataset-at-a-time. This means, that # while you may have an arbitrary number of iterative data writes, the write is performed in order. # In the future we may use a queing process to enable the simultaneous processing of multiple iterative writes at @@ -172,7 +172,7 @@ def iter_sin(chunk_length=10, max_chunks=100): # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # -from pynwb.form.data_utils import DataChunkIterator +from hdmf.data_utils import DataChunkIterator data = DataChunkIterator(data=iter_sin(10)) @@ -201,22 +201,22 @@ def iter_sin(chunk_length=10, max_chunks=100): # # maxshape=(None, 10), recommended_data_shape=(1, 10), dtype=float64 # -# As we can see :py:class:`~pynwb.form.data_utils.DataChunkIterator` automatically recommends +# As we can see :py:class:`~hdmf.data_utils.DataChunkIterator` automatically recommends # in its ``maxshape`` that the first dimensions of our array should be unlimited (``None``) and the second -# dimension be ``10`` (i.e., the length of our chunk. Since :py:class:`~pynwb.form.data_utils.DataChunkIterator` +# dimension be ``10`` (i.e., the length of our chunk. Since :py:class:`~hdmf.data_utils.DataChunkIterator` # has no way of knowing the minimum size of the array it automatically recommends the size of the first # chunk as the minimum size (i.e, ``(1, 10)``) and also infers the data type automatically from the first chunk. # To further customize this behavior we may also define the ``maxshape``, ``dtype``, and ``buffer_size`` when -# we create the :py:class:`~pynwb.form.data_utils.DataChunkIterator`. +# we create the :py:class:`~hdmf.data_utils.DataChunkIterator`. # # .. tip:: # -# We here used :py:class:`~pynwb.form.data_utils.DataChunkIterator` to conveniently wrap our data stream. -# :py:class:`~pynwb.form.data_utils.DataChunkIterator` assumes that our generators yields in **consecutive order** +# We here used :py:class:`~hdmf.data_utils.DataChunkIterator` to conveniently wrap our data stream. +# :py:class:`~hdmf.data_utils.DataChunkIterator` assumes that our generators yields in **consecutive order** # **single** complete element along the **first dimension** of our a array (i.e., iterate over the first # axis and yield one-element-at-a-time). This behavior is useful in many practical cases. However, if # this strategy does not match our needs, then you can alternatively implement our own derived -# :py:class:`~pynwb.form.data_utils.AbstractDataChunkIterator`. We show an example of this next. +# :py:class:`~hdmf.data_utils.AbstractDataChunkIterator`. We show an example of this next. # @@ -227,7 +227,7 @@ def iter_sin(chunk_length=10, max_chunks=100): # Step 1: Create a data chunk iterator for our sparse matrix # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -from pynwb.form.data_utils import AbstractDataChunkIterator, DataChunk +from hdmf.data_utils import AbstractDataChunkIterator, DataChunk class SparseMatrixIterator(AbstractDataChunkIterator): @@ -306,8 +306,8 @@ def maxshape(self): ##################### # In order to also enable compression and other advanced HDF5 dataset I/O featurs we can then also -# wrap our data via :py:class:`~pynwb.form.backends.hdf5.h5_utils.H5DataIO`. -from pynwb.form.backends.hdf5.h5_utils import H5DataIO +# wrap our data via :py:class:`~hdmf.backends.hdf5.h5_utils.H5DataIO`. +from hdmf.backends.hdf5.h5_utils import H5DataIO matrix2 = SparseMatrixIterator(shape=(xsize, ysize), num_chunks=num_chunks, chunk_shape=chunk_shape) @@ -318,7 +318,7 @@ def maxshape(self): ###################### # We can now also customize the chunking , fillvalue and other settings # -from pynwb.form.backends.hdf5.h5_utils import H5DataIO +from hdmf.backends.hdf5.h5_utils import H5DataIO # Increase the chunk size and add compression matrix3 = SparseMatrixIterator(shape=(xsize, ysize), @@ -427,7 +427,7 @@ def maxshape(self): # # **Advantages:** # -# * We only need to hold one :py:class:`~pynwb.form.data_utils.DataChunk` in memory at any given time +# * We only need to hold one :py:class:`~hdmf.data_utils.DataChunk` in memory at any given time # * Only the data chunks in the HDF5 file that contain non-default values are ever being allocated # * The overall size of our file is reduced significantly # * Reduced I/O load @@ -437,7 +437,7 @@ def maxshape(self): # # With great power comes great responsibility **!** I/O and storage cost will depend among others on the chunk size, # compression options, and the write pattern, i.e., the number and structure of the -# :py:class:`~pynwb.form.data_utils.DataChunk` objects written. For example, using ``(1,1)`` chunks and writing them +# :py:class:`~hdmf.data_utils.DataChunk` objects written. For example, using ``(1,1)`` chunks and writing them # one value at a time would result in poor I/O performance in most practical cases, because of the large number of # chunks and large number of small I/O operations required. # @@ -489,7 +489,7 @@ def maxshape(self): # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Note, we here use a generator for simplicity but we could equally well also implement our own -# :py:class:`~pynwb.form.data_utils.AbstractDataChunkIterator`. +# :py:class:`~hdmf.data_utils.AbstractDataChunkIterator`. def iter_largearray(filename, shape, dtype='float64'): @@ -510,7 +510,7 @@ def iter_largearray(filename, shape, dtype='float64'): # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # -from pynwb.form.data_utils import DataChunkIterator +from hdmf.data_utils import DataChunkIterator data = DataChunkIterator(data=iter_largearray(filename='basic_sparse_iterwrite_testdata.npy', shape=datashape), @@ -530,8 +530,8 @@ def iter_largearray(filename, shape, dtype='float64'): # .. tip:: # # Again, if we want to explicitly control how our data will be chunked (compressed etc.) -# in the HDF5 file then we need to wrap our :py:class:`~pynwb.form.data_utils.DataChunkIterator` -# using :py:class:`~pynwb.form.backends.hdf5.h5_utils.H5DataIO` +# in the HDF5 file then we need to wrap our :py:class:`~hdmf.data_utils.DataChunkIterator` +# using :py:class:`~hdmf.backends.hdf5.h5_utils.H5DataIO` #################### # Discussion @@ -589,7 +589,7 @@ def iter_largearray(filename, shape, dtype='float64'): # Step 1: Create a data chunk iterator for our multifile array # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -from pynwb.form.data_utils import AbstractDataChunkIterator, DataChunk # noqa +from hdmf.data_utils import AbstractDataChunkIterator, DataChunk # noqa class MultiFileArrayIterator(AbstractDataChunkIterator): @@ -666,8 +666,8 @@ def maxshape(self): # # Common mistakes that will result in errors on write: # -# * The size of a :py:class:`~pynwb.form.data_utils.DataChunk` does not match the selection. -# * The selection for the :py:class:`~pynwb.form.data_utils.DataChunk` is not supported by h5py +# * The size of a :py:class:`~hdmf.data_utils.DataChunk` does not match the selection. +# * The selection for the :py:class:`~hdmf.data_utils.DataChunk` is not supported by h5py # (e.g., unordered lists etc.) # # Other common mistakes: @@ -675,7 +675,7 @@ def maxshape(self): # * Choosing inappropriate chunk sizes. This typically means bad performance with regard to I/O and/or storage cost. # * Using auto chunking without supplying a good recommended_data_shape. h5py auto chunking can only make a good # guess of what the chunking should be if it (at least roughly) knows what the shape of the array will be. -# * Trying to wrap a data generator using the default :py:class:`~pynwb.form.data_utils.DataChunkIterator` +# * Trying to wrap a data generator using the default :py:class:`~hdmf.data_utils.DataChunkIterator` # when the generator does not comply with the assumptions of the default implementation (i.e., yield # individual, complete elements along the first dimension of the array one-at-a-time). Depending on the generator, # this may or may not result in an error on write, but the array you are generating will probably end up diff --git a/docs/gallery/general/linking_data.py b/docs/gallery/general/linking_data.py index 0a8f628e5..185317f58 100644 --- a/docs/gallery/general/linking_data.py +++ b/docs/gallery/general/linking_data.py @@ -151,9 +151,9 @@ # our TimeSeries, this means that :py:class:`~pynwb.NWBHDF5IO` will need to # determine on write how to treat the dataset. We can make this explicit and customize this # behavior on a per-dataset basis by wrapping our dataset using -# :py:meth:`~pynwb.form.backends.hdf5.h5_utils.H5DataIO` +# :py:meth:`~hdmf.backends.hdf5.h5_utils.H5DataIO` -from pynwb.form.backends.hdf5.h5_utils import H5DataIO +from hdmf.backends.hdf5.h5_utils import H5DataIO # Create another timeseries that links to the same data test_ts5 = TimeSeries(name='test_timeseries5', @@ -188,7 +188,7 @@ # --------------------------- # # Appending to files and linking is made possible by passing around the same -# :py:class:`~pynwb.form.build.map.BuildManager`. You can get a manager to pass around +# :py:class:`~hdmf.build.map.BuildManager`. You can get a manager to pass around # using the :py:meth:`~pynwb.get_manager` function. # @@ -245,5 +245,5 @@ # # External links are convenient but to share data we may want to hand a single file with all the # data to our collaborator rather than having to collect all relevant files. To do this, -# :py:class:`~pynwb.form.backends.hdf5.h5tools.HDF5IO` (and in turn :py:class:`~pynwb.NWBHDF5IO`) -# provide the convenience function :py:func:`~pynwb.form.backends.hdf5.h5tools.HDF5IO.copy_file` +# :py:class:`~hdmf.backends.hdf5.h5tools.HDF5IO` (and in turn :py:class:`~pynwb.NWBHDF5IO`) +# provide the convenience function :py:func:`~hdmf.backends.hdf5.h5tools.HDF5IO.copy_file` diff --git a/requirements-dev.txt b/requirements-dev.txt index a8e5b47a3..33da964d0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -certifi==2018.1.18 +certifi==2019.3.9 chardet==3.0.4 codecov==2.0.15 configparser==3.5.0 diff --git a/requirements.txt b/requirements.txt index 8629d0d6c..a597b6d9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -certifi==2018.1.18 +certifi==2019.3.9 chardet==3.0.4 h5py==2.9.0 idna==2.6 @@ -9,3 +9,4 @@ ruamel.yaml==0.15.85 six==1.11.0 urllib3==1.23 pandas==0.23.4 +hdmf==1.0.1 diff --git a/setup.py b/setup.py index 305032e75..3ae0203c4 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from setuptools import setup, find_packages +import re import versioneer @@ -12,6 +13,12 @@ schema_dir = 'data' +reqs_re = re.compile("[<=>]+") +with open('requirements.txt', 'r') as fp: + reqs = [reqs_re.split(x.strip())[0] for x in fp.readlines()] + +print(reqs) + setup_args = { 'name': 'pynwb', 'version': versioneer.get_version(), @@ -23,16 +30,7 @@ 'author_email': 'ajtritt@lbl.gov', 'url': 'https://github.com/NeurodataWithoutBorders/pynwb', 'license': "BSD", - 'install_requires': - [ - 'numpy', - 'pandas', - 'h5py', - 'ruamel.yaml', - 'python-dateutil', - 'six', - 'requests' - ], + 'install_requires': reqs, 'packages': pkgs, 'package_dir': {'': 'src'}, 'package_data': {'pynwb': ["%s/*.yaml" % schema_dir, "%s/*.json" % schema_dir]}, diff --git a/src/pynwb/__init__.py b/src/pynwb/__init__.py index 1006df61e..086f2bc7c 100644 --- a/src/pynwb/__init__.py +++ b/src/pynwb/__init__.py @@ -9,12 +9,12 @@ CORE_NAMESPACE = 'core' -from .form.spec import NamespaceCatalog # noqa: E402 -from .form.utils import docval, getargs, popargs, call_docval_func # noqa: E402 -from .form.backends.io import FORMIO # noqa: E402 -from .form.backends.hdf5 import HDF5IO as _HDF5IO # noqa: E402 -from .form.validate import ValidatorMap # noqa: E402 -from .form.build import BuildManager # noqa: E402 +from hdmf.spec import NamespaceCatalog # noqa: E402 +from hdmf.utils import docval, getargs, popargs, call_docval_func # noqa: E402 +from hdmf.backends.io import HDMFIO # noqa: E402 +from hdmf.backends.hdf5 import HDF5IO as _HDF5IO # noqa: E402 +from hdmf.validate import ValidatorMap # noqa: E402 +from hdmf.build import BuildManager # noqa: E402 from .spec import NWBDatasetSpec, NWBGroupSpec, NWBNamespace # noqa: E402 @@ -40,7 +40,7 @@ def _get_resources(): __NS_CATALOG = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace) -from .form.build import TypeMap as TypeMap # noqa: E402 +from hdmf.build import TypeMap as TypeMap # noqa: E402 __TYPE_MAP = TypeMap(__NS_CATALOG) @@ -171,8 +171,8 @@ def get_class(**kwargs): return __TYPE_MAP.get_container_cls(namespace, neurodata_type) -@docval({'name': 'io', 'type': FORMIO, - 'doc': 'the FORMIO object to read from'}, +@docval({'name': 'io', 'type': HDMFIO, + 'doc': 'the HDMFIO object to read from'}, {'name': 'namespace', 'type': str, 'doc': 'the namespace to validate against', 'default': CORE_NAMESPACE}, returns="errors in the file", rtype=list, diff --git a/src/pynwb/base.py b/src/pynwb/base.py index ef0232550..cf4897236 100644 --- a/src/pynwb/base.py +++ b/src/pynwb/base.py @@ -2,8 +2,8 @@ from collections import Iterable -from .form.utils import docval, getargs, popargs, fmt_docval_args, call_docval_func -from .form.data_utils import AbstractDataChunkIterator, DataIO +from hdmf.utils import docval, getargs, popargs, fmt_docval_args, call_docval_func +from hdmf.data_utils import AbstractDataChunkIterator, DataIO from . import register_class, CORE_NAMESPACE from .core import NWBDataInterface, MultiContainerInterface, NWBData diff --git a/src/pynwb/behavior.py b/src/pynwb/behavior.py index 27549c5ef..aa63a53f4 100644 --- a/src/pynwb/behavior.py +++ b/src/pynwb/behavior.py @@ -1,6 +1,6 @@ from collections import Iterable -from .form.utils import docval, popargs +from hdmf.utils import docval, popargs from . import register_class, CORE_NAMESPACE from .core import NWBContainer, MultiContainerInterface diff --git a/src/pynwb/core.py b/src/pynwb/core.py index 30f87b7dd..41d3cc01d 100644 --- a/src/pynwb/core.py +++ b/src/pynwb/core.py @@ -2,8 +2,8 @@ import numpy as np import pandas as pd -from .form.utils import docval, getargs, ExtenderMeta, call_docval_func, popargs, get_docval, fmt_docval_args, pystr -from .form import Container, Data, DataRegion, get_region_slicer +from hdmf.utils import docval, getargs, ExtenderMeta, call_docval_func, popargs, get_docval, fmt_docval_args, pystr +from hdmf import Container, Data, DataRegion, get_region_slicer from . import CORE_NAMESPACE, register_class from six import with_metaclass diff --git a/src/pynwb/device.py b/src/pynwb/device.py index 4a79a7e1a..4cd164eb5 100644 --- a/src/pynwb/device.py +++ b/src/pynwb/device.py @@ -1,4 +1,4 @@ -from .form.utils import docval, call_docval_func +from hdmf.utils import docval, call_docval_func from . import register_class, CORE_NAMESPACE from .core import NWBContainer diff --git a/src/pynwb/ecephys.py b/src/pynwb/ecephys.py index c2972dc3e..787c0f41d 100644 --- a/src/pynwb/ecephys.py +++ b/src/pynwb/ecephys.py @@ -1,8 +1,8 @@ import numpy as np from collections import Iterable -from .form.utils import docval, getargs, popargs, call_docval_func -from .form.data_utils import DataChunkIterator, assertEqualShape +from hdmf.utils import docval, getargs, popargs, call_docval_func +from hdmf.data_utils import DataChunkIterator, assertEqualShape from . import register_class, CORE_NAMESPACE from .base import TimeSeries, _default_resolution, _default_conversion diff --git a/src/pynwb/epoch.py b/src/pynwb/epoch.py index 6c3294af9..25a36bfc1 100644 --- a/src/pynwb/epoch.py +++ b/src/pynwb/epoch.py @@ -1,7 +1,7 @@ from bisect import bisect_left -from .form.utils import docval, getargs, popargs, call_docval_func -from .form.data_utils import DataIO +from hdmf.utils import docval, getargs, popargs, call_docval_func +from hdmf.data_utils import DataIO from . import register_class, CORE_NAMESPACE from .base import TimeSeries diff --git a/src/pynwb/file.py b/src/pynwb/file.py index 3fdca7b1a..b6f92fb98 100644 --- a/src/pynwb/file.py +++ b/src/pynwb/file.py @@ -4,8 +4,8 @@ from warnings import warn import copy as _copy -from .form.utils import docval, getargs, fmt_docval_args, call_docval_func, get_docval -from .form import Container +from hdmf.utils import docval, getargs, fmt_docval_args, call_docval_func, get_docval +from hdmf import Container from . import register_class, CORE_NAMESPACE from .base import TimeSeries, ProcessingModule diff --git a/src/pynwb/form/__init__.py b/src/pynwb/form/__init__.py deleted file mode 100644 index 1def60f42..000000000 --- a/src/pynwb/form/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# flake8: noqa: F401 -from .container import Container, Data, DataRegion -from .utils import docval, getargs -from .data_utils import ListSlicer -from .backends.hdf5.h5_utils import H5RegionSlicer, H5Dataset -from . import query - - -@docval({'name': 'dataset', 'type': None, 'doc': 'the HDF5 dataset to slice'}, - {'name': 'region', 'type': None, 'doc': 'the region reference to use to slice'}, - is_method=False) -def get_region_slicer(**kwargs): - dataset, region = getargs('dataset', 'region', kwargs) - if isinstance(dataset, (list, tuple, Data)): - return ListSlicer(dataset, region) - elif isinstance(dataset, H5Dataset): - return H5RegionSlicer(dataset, region) - return None diff --git a/src/pynwb/form/array.py b/src/pynwb/form/array.py deleted file mode 100644 index 804a01be4..000000000 --- a/src/pynwb/form/array.py +++ /dev/null @@ -1,197 +0,0 @@ -import numpy as np -from abc import abstractmethod, ABCMeta -from six import with_metaclass - - -class Array(object): - - def __init__(self, data): - self.__data = data - if hasattr(data, 'dtype'): - self.dtype = data.dtype - else: - tmp = data - while isinstance(tmp, (list, tuple)): - tmp = tmp[0] - self.dtype = type(tmp) - - @property - def data(self): - return self.__data - - def __len__(self): - return len(self.__data) - - def get_data(self): - return self.__data - - def __getidx__(self, arg): - return self.__data[arg] - - def __sliceiter(self, arg): - return (x for x in range(*arg.indices(len(self)))) - - def __getitem__(self, arg): - if isinstance(arg, list): - idx = list() - for i in arg: - if isinstance(i, slice): - idx.extend(x for x in self.__sliceiter(i)) - else: - idx.append(i) - return np.fromiter((self.__getidx__(x) for x in idx), dtype=self.dtype) - elif isinstance(arg, slice): - return np.fromiter((self.__getidx__(x) for x in self.__sliceiter(arg)), dtype=self.dtype) - elif isinstance(arg, tuple): - return (self.__getidx__(arg[0]), self.__getidx__(arg[1])) - else: - return self.__getidx__(arg) - - -class AbstractSortedArray(with_metaclass(ABCMeta, Array)): - ''' - An abstract class for representing sorted array - ''' - - @abstractmethod - def find_point(self, val): - pass - - def get_data(self): - return self - - def __lower(self, other): - ins = self.find_point(other) - return ins - - def __upper(self, other): - ins = self.__lower(other) - while self[ins] == other: - ins += 1 - return ins - - def __lt__(self, other): - ins = self.__lower(other) - return slice(0, ins) - - def __le__(self, other): - ins = self.__upper(other) - return slice(0, ins) - - def __gt__(self, other): - ins = self.__upper(other) - return slice(ins, len(self)) - - def __ge__(self, other): - ins = self.__lower(other) - return slice(ins, len(self)) - - @staticmethod - def __sort(a): - if isinstance(a, tuple): - return a[0] - else: - return a - - def __eq__(self, other): - if isinstance(other, list): - ret = list() - for i in other: - eq = self == i - ret.append(eq) - ret = sorted(ret, key=self.__sort) - tmp = list() - for i in range(1, len(ret)): - a, b = ret[i-1], ret[i] - if isinstance(a, tuple): - if isinstance(b, tuple): - if a[1] >= b[0]: - b[0] = a[0] - else: - tmp.append(slice(*a)) - else: - if b > a[1]: - tmp.append(slice(*a)) - elif b == a[1]: - a[1] == b+1 - else: - ret[i] = a - else: - if isinstance(b, tuple): - if a < b[0]: - tmp.append(a) - else: - if b - a == 1: - ret[i] = (a, b) - else: - tmp.append(a) - if isinstance(ret[-1], tuple): - tmp.append(slice(*ret[-1])) - else: - tmp.append(ret[-1]) - ret = tmp - return ret - elif isinstance(other, tuple): - ge = self >= other[0] - ge = ge.start - lt = self < other[1] - lt = lt.stop - if ge == lt: - return ge - else: - return slice(ge, lt) - else: - lower = self.__lower(other) - upper = self.__upper(other) - d = upper - lower - if d == 1: - return lower - elif d == 0: - return None - else: - return slice(lower, upper) - - def __ne__(self, other): - eq = self == other - if isinstance(eq, tuple): - return [slice(0, eq[0]), slice(eq[1], len(self))] - else: - return [slice(0, eq), slice(eq+1, len(self))] - - -class SortedArray(AbstractSortedArray): - ''' - A class for wrapping sorted arrays. This class overrides - <,>,<=,>=,==, and != to leverage the sorted content for - efficiency. - ''' - - def __init__(self, array): - super(SortedArray, self).__init__(array) - - def find_point(self, val): - return np.searchsorted(self.data, val) - - -class LinSpace(SortedArray): - - def __init__(self, start, stop, step): - self.start = start - self.stop = stop - self.step = step - self.dtype = float if any(isinstance(s, float) for s in (start, stop, step)) else int - self.__len = int((stop - start)/step) - - def __len__(self): - return self.__len - - def find_point(self, val): - nsteps = (val-self.start)/self.step - fl = int(nsteps) - if fl == nsteps: - return int(fl) - else: - return int(fl+1) - - def __getidx__(self, arg): - return self.start + self.step*arg diff --git a/src/pynwb/form/backends/__init__.py b/src/pynwb/form/backends/__init__.py deleted file mode 100644 index db348307c..000000000 --- a/src/pynwb/form/backends/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# flake8: noqa: F401 -from . import hdf5 diff --git a/src/pynwb/form/backends/hdf5/__init__.py b/src/pynwb/form/backends/hdf5/__init__.py deleted file mode 100644 index dbc2f32ae..000000000 --- a/src/pynwb/form/backends/hdf5/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# flake8: noqa: F401 -from . import h5_utils -from .h5tools import HDF5IO -from .h5_utils import H5RegionSlicer, H5DataIO -from . import h5tools -from .h5tools import H5SpecWriter -from .h5tools import H5SpecReader diff --git a/src/pynwb/form/backends/hdf5/h5_utils.py b/src/pynwb/form/backends/hdf5/h5_utils.py deleted file mode 100644 index 144c4df90..000000000 --- a/src/pynwb/form/backends/hdf5/h5_utils.py +++ /dev/null @@ -1,294 +0,0 @@ -from copy import copy -from collections import Iterable -from six import binary_type, text_type -from h5py import Group, Dataset, RegionReference, Reference, special_dtype -import json -import h5py -import numpy as np -import warnings -import os - -from ...query import FORMDataset -from ...array import Array -from ...utils import docval, getargs, popargs, call_docval_func -from ...data_utils import RegionSlicer, DataIO, AbstractDataChunkIterator - -from ...spec import SpecWriter, SpecReader - - -class H5Dataset(FORMDataset): - @docval({'name': 'dataset', 'type': (Dataset, Array), 'doc': 'the HDF5 file lazily evaluate'}, - {'name': 'io', 'type': 'HDF5IO', 'doc': 'the IO object that was used to read the underlying dataset'}) - def __init__(self, **kwargs): - self.__io = popargs('io', kwargs) - call_docval_func(super(H5Dataset, self).__init__, kwargs) - - @property - def io(self): - return self.__io - - @property - def regionref(self): - return self.dataset.regionref - - @property - def ref(self): - return self.dataset.ref - - -class H5TableDataset(H5Dataset): - - @docval({'name': 'dataset', 'type': (Dataset, Array), 'doc': 'the HDF5 file lazily evaluate'}, - {'name': 'io', 'type': 'HDF5IO', 'doc': 'the IO object that was used to read the underlying dataset'}, - {'name': 'types', 'type': (list, tuple), - 'doc': 'the IO object that was used to read the underlying dataset'}) - def __init__(self, **kwargs): - types = popargs('types', kwargs) - call_docval_func(super(H5TableDataset, self).__init__, kwargs) - self.__refgetters = dict() - for i, t in enumerate(types): - if t is RegionReference: - self.__refgetters[i] = self.__get_regref - elif t is Reference: - self.__refgetters[i] = self.__get_ref - tmp = list() - for i in range(len(self.dataset.dtype)): - sub = self.dataset.dtype[i] - if sub.metadata: - if 'vlen' in sub.metadata: - t = sub.metadata['vlen'] - if t is text_type: - tmp.append('utf') - elif t is binary_type: - tmp.append('ascii') - elif 'ref' in sub.metadata: - t = sub.metadata['ref'] - if t is Reference: - tmp.append('object') - elif t is RegionReference: - tmp.append('region') - else: - tmp.append(sub.type.__name__) - self.__dtype = tmp - - @property - def dtype(self): - return self.__dtype - - def __getitem__(self, arg): - rows = copy(super(H5TableDataset, self).__getitem__(arg)) - if isinstance(arg, int): - self.__swap_refs(rows) - else: - for row in rows: - self.__swap_refs(row) - return rows - - def __swap_refs(self, row): - for i in self.__refgetters: - getref = self.__refgetters[i] - row[i] = getref(row[i]) - - def __get_ref(self, ref): - return self.io.get_container(self.dataset.file[ref]) - - def __get_regref(self, ref): - obj = self.__get_ref(ref) - return obj[ref] - - -class H5ReferenceDataset(H5Dataset): - - def __getitem__(self, arg): - ref = super(H5ReferenceDataset, self).__getitem__(arg) - if isinstance(ref, np.ndarray): - return [self.io.get_container(self.dataset.file[x]) for x in ref] - else: - return self.io.get_container(self.dataset.file[ref]) - - @property - def dtype(self): - return 'object' - - -class H5RegionDataset(H5ReferenceDataset): - - def __getitem__(self, arg): - obj = super(H5RegionDataset, self).__getitem__(arg) - ref = self.dataset[arg] - return obj[ref] - - @property - def dtype(self): - return 'region' - - -class H5SpecWriter(SpecWriter): - - __str_type = special_dtype(vlen=binary_type) - - @docval({'name': 'group', 'type': Group, 'doc': 'the HDF5 file to write specs to'}) - def __init__(self, **kwargs): - self.__group = getargs('group', kwargs) - - @staticmethod - def stringify(spec): - ''' - Converts a spec into a JSON string to write to a dataset - ''' - return json.dumps(spec, separators=(',', ':')) - - def __write(self, d, name): - data = self.stringify(d) - dset = self.__group.create_dataset(name, data=data, dtype=self.__str_type) - return dset - - def write_spec(self, spec, path): - return self.__write(spec, path) - - def write_namespace(self, namespace, path): - return self.__write({'namespaces': [namespace]}, path) - - -class H5SpecReader(SpecReader): - - @docval({'name': 'group', 'type': Group, 'doc': 'the HDF5 file to read specs from'}) - def __init__(self, **kwargs): - self.__group = getargs('group', kwargs) - super_kwargs = {'source': "%s:%s" % (os.path.abspath(self.__group.file.name), self.__group.name)} - call_docval_func(super(H5SpecReader, self).__init__, super_kwargs) - - def __read(self, path): - s = self.__group[path][()] - if isinstance(s, bytes): - s = s.decode('UTF-8') - d = json.loads(s) - return d - - def read_spec(self, spec_path): - return self.__read(spec_path) - - def read_namespace(self, ns_path): - ret = self.__read(ns_path) - ret = ret['namespaces'] - return ret - - -class H5RegionSlicer(RegionSlicer): - - @docval({'name': 'dataset', 'type': (Dataset, H5Dataset), 'doc': 'the HDF5 dataset to slice'}, - {'name': 'region', 'type': RegionReference, 'doc': 'the region reference to use to slice'}) - def __init__(self, **kwargs): - self.__dataset = getargs('dataset', kwargs) - self.__regref = getargs('region', kwargs) - self.__len = self.__dataset.regionref.selection(self.__regref)[0] - self.__region = None - - def __read_region(self): - if self.__region is None: - self.__region = self.__dataset[self.__regref] - - def __getitem__(self, idx): - self.__read_region() - return self.__region[idx] - - def __len__(self): - return self.__len - - -class H5DataIO(DataIO): - """ - Wrap data arrays for write via HDF5IO to customize I/O behavior, such as compression and chunking - for data arrays. - """ - - @docval({'name': 'data', - 'type': (np.ndarray, list, tuple, h5py.Dataset, Iterable), - 'doc': 'the data to be written. NOTE: If an h5py.Dataset is used, all other settings but link_data' + - ' will be ignored as the dataset will either be linked to or copied as is in H5DataIO.'}, - {'name': 'maxshape', - 'type': tuple, - 'doc': 'Dataset will be resizable up to this shape (Tuple). Automatically enables chunking.' + - 'Use None for the axes you want to be unlimited.', - 'default': None}, - {'name': 'chunks', - 'type': (bool, tuple), - 'doc': 'Chunk shape or True ti enable auto-chunking', - 'default': None}, - {'name': 'compression', - 'type': (str, bool), - 'doc': 'Compression strategy. If a bool is given, then gzip compression will be used by default.' + - 'http://docs.h5py.org/en/latest/high/dataset.html#dataset-compression', - 'default': None}, - {'name': 'compression_opts', - 'type': int, - 'doc': 'Parameter for compression filter', - 'default': None}, - {'name': 'fillvalue', - 'type': None, - 'doc': 'Value to eb returned when reading uninitialized parts of the dataset', - 'default': None}, - {'name': 'shuffle', - 'type': bool, - 'doc': 'Enable shuffle I/O filter. http://docs.h5py.org/en/latest/high/dataset.html#dataset-shuffle', - 'default': None}, - {'name': 'fletcher32', - 'type': bool, - 'doc': 'Enable fletcher32 checksum. http://docs.h5py.org/en/latest/high/dataset.html#dataset-fletcher32', - 'default': None}, - {'name': 'link_data', - 'type': bool, - 'doc': 'If data is an h5py.Dataset should it be linked to or copied. NOTE: This parameter is only ' + - 'allowed if data is an h5py.Dataset', - 'default': False} - ) - def __init__(self, **kwargs): - # Get the list of I/O options that user has passed in - ioarg_names = [name for name in kwargs.keys() if name not in['data', 'link_data']] - # Remove the ioargs from kwargs - ioarg_values = [popargs(argname, kwargs) for argname in ioarg_names] - # Consume link_data parameter - self.__link_data = popargs('link_data', kwargs) - # Check for possible collision with other parameters - if not isinstance(getargs('data', kwargs), Dataset) and self.__link_data: - self.__link_data = False - warnings.warn('link_data parameter in H5DataIO will be ignored') - # Call the super constructor and consume the data parameter - call_docval_func(super(H5DataIO, self).__init__, kwargs) - # Construct the dict with the io args, ignoring all options that were set to None - self.__iosettings = {k: v for k, v in zip(ioarg_names, ioarg_values) if v is not None} - # Set io_properties for DataChunkIterators - if isinstance(self.data, AbstractDataChunkIterator): - # Define the chunking options if the user has not set them explicitly. - if 'chunks' not in self.__iosettings and self.data.recommended_chunk_shape() is not None: - self.__iosettings['chunks'] = self.data.recommended_chunk_shape() - # Define the maxshape of the data if not provided by the user - if 'maxshape' not in self.__iosettings: - self.__iosettings['maxshape'] = self.data.maxshape - if 'compression' in self.__iosettings: - if isinstance(self.__iosettings['compression'], bool): - if self.__iosettings['compression']: - self.__iosettings['compression'] = 'gzip' - else: - self.__iosettings.pop('compression', None) - if 'compression_opts' in self.__iosettings: - warnings.warn('Compression disabled by compression=False setting. ' + - 'compression_opts parameter will, therefore, be ignored.') - self.__iosettings.pop('compression_opts', None) - if 'compression' in self.__iosettings: - if self.__iosettings['compression'] != 'gzip': - warnings.warn(str(self.__iosettings['compression']) + " compression may not be available" + - "on all installations of HDF5. Use of gzip is recommended to ensure portability of" + - "the generated HDF5 files.") - # Check possible parameter collisions - if isinstance(self.data, Dataset): - for k in self.__iosettings.keys(): - warnings.warn("%s in H5DataIO will be ignored with H5DataIO.data being an HDF5 dataset" % k) - - @property - def link_data(self): - return self.__link_data - - @property - def io_settings(self): - return self.__iosettings diff --git a/src/pynwb/form/backends/hdf5/h5tools.py b/src/pynwb/form/backends/hdf5/h5tools.py deleted file mode 100644 index 3b1039cd1..000000000 --- a/src/pynwb/form/backends/hdf5/h5tools.py +++ /dev/null @@ -1,938 +0,0 @@ -from collections import deque -import numpy as np -import os.path -from functools import partial -from h5py import File, Group, Dataset, special_dtype, SoftLink, ExternalLink, Reference, RegionReference, check_dtype -from six import raise_from, text_type, string_types, binary_type -import warnings -from ...container import Container - -from ...utils import docval, getargs, popargs, call_docval_func -from ...data_utils import AbstractDataChunkIterator, get_shape -from ...build import Builder, GroupBuilder, DatasetBuilder, LinkBuilder, BuildManager,\ - RegionBuilder, ReferenceBuilder, TypeMap -from ...spec import RefSpec, DtypeSpec, NamespaceCatalog, GroupSpec -from ...spec import NamespaceBuilder - -from .h5_utils import H5ReferenceDataset, H5RegionDataset, H5TableDataset,\ - H5DataIO, H5SpecReader, H5SpecWriter - -from ..io import FORMIO - -ROOT_NAME = 'root' -SPEC_LOC_ATTR = '.specloc' -H5_TEXT = special_dtype(vlen=text_type) -H5_BINARY = special_dtype(vlen=binary_type) -H5_REF = special_dtype(ref=Reference) -H5_REGREF = special_dtype(ref=RegionReference) - - -class HDF5IO(FORMIO): - - @docval({'name': 'path', 'type': str, 'doc': 'the path to the HDF5 file'}, - {'name': 'manager', 'type': BuildManager, 'doc': 'the BuildManager to use for I/O', 'default': None}, - {'name': 'mode', 'type': str, - 'doc': 'the mode to open the HDF5 file with, one of ("w", "r", "r+", "a", "w-")'}, - {'name': 'comm', 'type': 'Intracomm', - 'doc': 'the MPI communicator to use for parallel I/O', 'default': None}, - {'name': 'file', 'type': File, 'doc': 'a pre-existing h5py.File object', 'default': None}) - def __init__(self, **kwargs): - '''Open an HDF5 file for IO - - For `mode`, see `h5py.File _`. - ''' - path, manager, mode, comm, file_obj = popargs('path', 'manager', 'mode', 'comm', 'file', kwargs) - - if file_obj is not None and os.path.abspath(file_obj.filename) != os.path.abspath(path): - raise ValueError('You argued {} as this object\'s path, but supplied a file with filename: {}'.format()) - - if manager is None: - manager = BuildManager(TypeMap(NamespaceCatalog())) - self.__comm = comm - self.__mode = mode - self.__path = path - self.__file = file_obj - super(HDF5IO, self).__init__(manager, source=path) - self.__built = dict() # keep track of which files have been read - self.__read = dict() # keep track of each builder for each dataset/group/link - self.__ref_queue = deque() # a queue of the references that need to be added - - @property - def comm(self): - return self.__comm - - @property - def _file(self): - return self.__file - - @classmethod - @docval({'name': 'namespace_catalog', - 'type': (NamespaceCatalog, TypeMap), - 'doc': 'the NamespaceCatalog or TypeMap to load namespaces into'}, - {'name': 'path', 'type': str, 'doc': 'the path to the HDF5 file'}, - {'name': 'namespaces', 'type': list, 'doc': 'the namespaces to load', 'default': None}) - def load_namespaces(cls, namespace_catalog, path, namespaces=None): - ''' - Load cached namespaces from a file. - ''' - f = File(path, 'r') - if SPEC_LOC_ATTR not in f.attrs: - msg = "No cached namespaces found in %s" % path - warnings.warn(msg) - else: - spec_group = f[f.attrs[SPEC_LOC_ATTR]] - if namespaces is None: - namespaces = list(spec_group.keys()) - for ns in namespaces: - ns_group = spec_group[ns] - latest_version = list(ns_group.keys())[-1] - ns_group = ns_group[latest_version] - reader = H5SpecReader(ns_group) - namespace_catalog.load_namespaces('namespace', reader=reader) - f.close() - - @classmethod - def __convert_namespace(cls, ns_catalog, namespace): - ns = ns_catalog.get_namespace(namespace) - builder = NamespaceBuilder(ns.doc, ns.name, - full_name=ns.full_name, - version=ns.version, - author=ns.author, - contact=ns.contact) - for elem in ns.schema: - if 'namespace' in elem: - inc_ns = elem['namespace'] - builder.include_namespace(inc_ns) - else: - source = elem['source'] - for dt in ns_catalog.get_types(source): - spec = ns_catalog.get_spec(namespace, dt) - if spec.parent is not None: - continue - h5_source = cls.__get_name(source) - spec = cls.__copy_spec(spec) - builder.add_spec(h5_source, spec) - return builder - - @classmethod - def __get_name(cls, path): - return os.path.splitext(path)[0] - - @classmethod - def __copy_spec(cls, spec): - kwargs = dict() - kwargs['attributes'] = cls.__get_new_specs(spec.attributes, spec) - to_copy = ['doc', 'name', 'default_name', 'linkable', 'quantity', spec.inc_key(), spec.def_key()] - if isinstance(spec, GroupSpec): - kwargs['datasets'] = cls.__get_new_specs(spec.datasets, spec) - kwargs['groups'] = cls.__get_new_specs(spec.groups, spec) - kwargs['links'] = cls.__get_new_specs(spec.links, spec) - else: - to_copy.append('dtype') - to_copy.append('shape') - to_copy.append('dims') - for key in to_copy: - val = getattr(spec, key) - if val is not None: - kwargs[key] = val - ret = spec.build_spec(kwargs) - return ret - - @classmethod - def __get_new_specs(cls, subspecs, spec): - ret = list() - for subspec in subspecs: - if not spec.is_inherited_spec(subspec) or spec.is_overridden_spec(subspec): - ret.append(subspec) - return ret - - @classmethod - @docval({'name': 'source_filename', 'type': str, 'doc': 'the path to the HDF5 file to copy'}, - {'name': 'dest_filename', 'type': str, 'doc': 'the name of the destination file'}, - {'name': 'expand_external', 'type': bool, 'doc': 'expand external links into new objects', 'default': True}, - {'name': 'expand_refs', 'type': bool, 'doc': 'copy objects which are pointed to by reference', - 'default': False}, - {'name': 'expand_soft', 'type': bool, 'doc': 'expand soft links into new objects', 'default': False} - ) - def copy_file(self, **kwargs): - """ - Convenience function to copy an HDF5 file while allowing external links to be resolved. - - NOTE: The source file will be opened in 'r' mode and the destination file will be opened in 'w' mode - using h5py. To avoid possible collisions, care should be taken that, e.g., the source file is - not opened already when calling this function. - - """ - source_filename, dest_filename, expand_external, expand_refs, expand_soft = getargs('source_filename', - 'dest_filename', - 'expand_external', - 'expand_refs', - 'expand_soft', - kwargs) - source_file = File(source_filename, 'r') - dest_file = File(dest_filename, 'w') - for objname in source_file["/"].keys(): - source_file.copy(source=objname, - dest=dest_file, - name=objname, - expand_external=expand_external, - expand_refs=expand_refs, - expand_soft=expand_soft, - shallow=False, - without_attrs=False, - ) - for objname in source_file['/'].attrs: - dest_file['/'].attrs[objname] = source_file['/'].attrs[objname] - source_file.close() - dest_file.close() - - @docval({'name': 'container', 'type': Container, 'doc': 'the Container object to write'}, - {'name': 'cache_spec', 'type': bool, 'doc': 'cache specification to file', 'default': False}, - {'name': 'link_data', 'type': bool, - 'doc': 'If not specified otherwise link (True) or copy (False) HDF5 Datasets', 'default': True}) - def write(self, **kwargs): - cache_spec = popargs('cache_spec', kwargs) - call_docval_func(super(HDF5IO, self).write, kwargs) - if cache_spec: - ref = self.__file.attrs.get(SPEC_LOC_ATTR) - spec_group = None - if ref is not None: - spec_group = self.__file[ref] - else: - path = 'specifications' # do something to figure out where the specifications should go - spec_group = self.__file.require_group(path) - self.__file.attrs[SPEC_LOC_ATTR] = spec_group.ref - ns_catalog = self.manager.namespace_catalog - for ns_name in ns_catalog.namespaces: - ns_builder = self.__convert_namespace(ns_catalog, ns_name) - namespace = ns_catalog.get_namespace(ns_name) - if namespace.version is None: - group_name = '%s/unversioned' % ns_name - else: - group_name = '%s/%s' % (ns_name, namespace.version) - ns_group = spec_group.require_group(group_name) - writer = H5SpecWriter(ns_group) - ns_builder.export('namespace', writer=writer) - - @docval(returns='a GroupBuilder representing the NWB Dataset', rtype='GroupBuilder') - def read_builder(self): - f_builder = self.__read.get(self.__file) - # ignore cached specs when reading builder - ignore = set() - specloc = self.__file.attrs.get(SPEC_LOC_ATTR) - if specloc is not None: - ignore.add(self.__file[specloc].name) - if f_builder is None: - f_builder = self.__read_group(self.__file, ROOT_NAME, ignore=ignore) - self.__read[self.__file] = f_builder - return f_builder - - def __set_built(self, fpath, path, builder): - self.__built.setdefault(fpath, dict()).setdefault(path, builder) - - def __get_built(self, fpath, path): - fdict = self.__built.get(fpath) - if fdict: - return fdict.get(path) - else: - return None - - @docval({'name': 'h5obj', 'type': (Dataset, Group), - 'doc': 'the HDF5 object to the corresponding Container/Data object for'}) - def get_container(self, **kwargs): - h5obj = getargs('h5obj', kwargs) - fpath = h5obj.file.filename - path = h5obj.name - builder = self.__get_built(fpath, path) - if builder is None: - msg = '%s:%s has not been built' % (fpath, path) - raise ValueError(msg) - container = self.manager.construct(builder) - return container - - def __read_group(self, h5obj, name=None, ignore=set()): - kwargs = { - "attributes": self.__read_attrs(h5obj), - "groups": dict(), - "datasets": dict(), - "links": dict() - } - - for key, val in kwargs['attributes'].items(): - if isinstance(val, bytes): - kwargs['attributes'][key] = val.decode('UTF-8') - - if name is None: - name = str(os.path.basename(h5obj.name)) - for k in h5obj: - sub_h5obj = h5obj.get(k) - if sub_h5obj.name in ignore: - continue - if not (sub_h5obj is None): - link_type = h5obj.get(k, getlink=True) - if isinstance(link_type, SoftLink) or isinstance(link_type, ExternalLink): - # Reading links might be better suited in its own function - # get path of link (the key used for tracking what's been built) - target_path = link_type.path - builder_name = os.path.basename(target_path) - # get builder if already read, else build it - builder = self.__get_built(sub_h5obj.file.filename, target_path) - if builder is None: - # NOTE: all links must have absolute paths - if isinstance(sub_h5obj, Dataset): - builder = self.__read_dataset(sub_h5obj, builder_name) - else: - builder = self.__read_group(sub_h5obj, builder_name, ignore=ignore) - self.__set_built(sub_h5obj.file.filename, target_path, builder) - link_builder = LinkBuilder(builder, k, source=self.__path) - link_builder.written = True - kwargs['links'][builder_name] = link_builder - else: - builder = self.__get_built(sub_h5obj.file.filename, sub_h5obj.name) - obj_type = None - read_method = None - if isinstance(sub_h5obj, Dataset): - read_method = self.__read_dataset - obj_type = kwargs['datasets'] - else: - read_method = partial(self.__read_group, ignore=ignore) - obj_type = kwargs['groups'] - if builder is None: - builder = read_method(sub_h5obj) - self.__set_built(sub_h5obj.file.filename, sub_h5obj.name, builder) - obj_type[builder.name] = builder - else: - warnings.warn('Broken Link: %s' % os.path.join(h5obj.name, k)) - kwargs['datasets'][k] = None - continue - kwargs['source'] = self.__path - ret = GroupBuilder(name, **kwargs) - ret.written = True - return ret - - def __read_dataset(self, h5obj, name=None): - kwargs = { - "attributes": self.__read_attrs(h5obj), - "dtype": h5obj.dtype, - "maxshape": h5obj.maxshape - } - for key, val in kwargs['attributes'].items(): - if isinstance(val, bytes): - kwargs['attributes'][key] = val.decode('UTF-8') - - if name is None: - name = str(os.path.basename(h5obj.name)) - kwargs['source'] = self.__path - ndims = len(h5obj.shape) - if ndims == 0: # read scalar - scalar = h5obj[()] - if isinstance(scalar, bytes): - scalar = scalar.decode('UTF-8') - - if isinstance(scalar, Reference): - # TODO (AJTRITT): This should call __read_ref to support Group references - target = h5obj.file[scalar] - target_builder = self.__read_dataset(target) - self.__set_built(target.file.filename, target.name, target_builder) - if isinstance(scalar, RegionReference): - kwargs['data'] = RegionBuilder(scalar, target_builder) - else: - kwargs['data'] = ReferenceBuilder(target_builder) - else: - kwargs["data"] = scalar - elif ndims == 1: - d = None - if h5obj.dtype.kind == 'O': # read list of strings or list of references - elem1 = h5obj[0] - if isinstance(elem1, (text_type, binary_type)): - d = h5obj[()] - elif isinstance(elem1, RegionReference): - d = H5RegionDataset(h5obj, self) - elif isinstance(elem1, Reference): - d = H5ReferenceDataset(h5obj, self) - elif h5obj.dtype.kind == 'V': # table - cpd_dt = h5obj.dtype - ref_cols = [check_dtype(ref=cpd_dt[i]) for i in range(len(cpd_dt))] - d = H5TableDataset(h5obj, self, ref_cols) - else: - d = h5obj - kwargs["data"] = d - else: - kwargs["data"] = h5obj - ret = DatasetBuilder(name, **kwargs) - ret.written = True - return ret - - def __read_attrs(self, h5obj): - ret = dict() - for k, v in h5obj.attrs.items(): - if k == SPEC_LOC_ATTR: # ignore cached spec - continue - if isinstance(v, RegionReference): - raise ValueError("cannot read region reference attributes yet") - elif isinstance(v, Reference): - ret[k] = self.__read_ref(h5obj.file[v]) - else: - ret[k] = v - return ret - - def __read_ref(self, h5obj): - ret = None - ret = self.__get_built(h5obj.file.filename, h5obj.name) - if ret is None: - if isinstance(h5obj, Dataset): - ret = self.__read_dataset(h5obj) - elif isinstance(h5obj, Group): - ret = self.__read_group(h5obj) - else: - raise ValueError("h5obj must be a Dataset or a Group - got %s" % str(h5obj)) - self.__set_built(h5obj.file.filename, h5obj.name, ret) - return ret - - def open(self): - if self.__file is None: - open_flag = self.__mode - self.__file = File(self.__path, open_flag) - - def close(self): - if self.__file is not None: - self.__file.close() - - @docval({'name': 'builder', 'type': GroupBuilder, 'doc': 'the GroupBuilder object representing the NWBFile'}, - {'name': 'link_data', 'type': bool, - 'doc': 'If not specified otherwise link (True) or copy (False) HDF5 Datasets', 'default': True}) - def write_builder(self, **kwargs): - f_builder, link_data = getargs('builder', 'link_data', kwargs) - for name, gbldr in f_builder.groups.items(): - self.write_group(self.__file, gbldr) - for name, dbldr in f_builder.datasets.items(): - self.write_dataset(self.__file, dbldr, link_data) - self.set_attributes(self.__file, f_builder.attributes) - self.__add_refs() - - def __add_refs(self): - ''' - Add all references in the file. - - References get queued to be added at the end of write. This is because - the current traversal algorithm (i.e. iterating over GroupBuilder items) - does not happen in a guaranteed order. We need to figure out what objects - will be references, and then write them after we write everything else. - ''' - failed = set() - while len(self.__ref_queue) > 0: - call = self.__ref_queue.popleft() - try: - call() - except KeyError: - if id(call) in failed: - raise RuntimeError('Unable to resolve reference') - failed.add(id(call)) - self.__ref_queue.append(call) - - @classmethod - def get_type(cls, data): - if isinstance(data, (text_type, string_types)): - return H5_TEXT - elif isinstance(data, Container): - return H5_REF - elif not hasattr(data, '__len__'): - return type(data) - else: - if len(data) == 0: - raise ValueError('cannot determine type for empty data') - return cls.get_type(data[0]) - - __dtypes = { - "float": np.float32, - "float32": np.float32, - "double": np.float64, - "float64": np.float64, - "long": np.int64, - "int64": np.int64, - "uint64": np.uint64, - "int": np.int32, - "int32": np.int32, - "int16": np.int16, - "int8": np.int8, - "bool": np.bool_, - "text": H5_TEXT, - "text": H5_TEXT, - "utf": H5_TEXT, - "utf8": H5_TEXT, - "utf-8": H5_TEXT, - "ascii": H5_BINARY, - "str": H5_BINARY, - "isodatetime": H5_TEXT, - "uint32": np.uint32, - "uint16": np.uint16, - "uint8": np.uint8, - "ref": H5_REF, - "reference": H5_REF, - "object": H5_REF, - "region": H5_REGREF, - } - - @classmethod - def __resolve_dtype__(cls, dtype, data): - # TODO: These values exist, but I haven't solved them yet - # binary - # number - dtype = cls.__resolve_dtype_helper__(dtype) - if dtype is None: - dtype = cls.get_type(data) - return dtype - - @classmethod - def __resolve_dtype_helper__(cls, dtype): - if dtype is None: - return None - elif isinstance(dtype, str): - return cls.__dtypes.get(dtype) - elif isinstance(dtype, dict): - return cls.__dtypes.get(dtype['reftype']) - else: - return np.dtype([(x['name'], cls.__resolve_dtype_helper__(x['dtype'])) for x in dtype]) - - @docval({'name': 'obj', 'type': (Group, Dataset), 'doc': 'the HDF5 object to add attributes to'}, - {'name': 'attributes', - 'type': dict, - 'doc': 'a dict containing the attributes on the Group or Dataset, indexed by attribute name'}) - def set_attributes(self, **kwargs): - obj, attributes = getargs('obj', 'attributes', kwargs) - for key, value in attributes.items(): - if isinstance(value, (set, list, tuple)): - tmp = tuple(value) - if len(tmp) > 0: - # a list of strings will need a special type - if isinstance(tmp[0], (text_type, binary_type)): - value = [np.string_(s) for s in tmp] - elif isinstance(tmp[0], Container): # a list of references - self.__queue_ref(self._make_attr_ref_filler(obj, key, tmp)) - else: - value = np.array(value) - obj.attrs[key] = value - elif isinstance(value, (Container, Builder, ReferenceBuilder)): # a reference - self.__queue_ref(self._make_attr_ref_filler(obj, key, value)) - else: - obj.attrs[key] = value # a regular scalar - - def _make_attr_ref_filler(self, obj, key, value): - ''' - Make the callable for setting references to attributes - ''' - if isinstance(value, (tuple, list)): - def _filler(): - ret = list() - for item in value: - ret.append(self.__get_ref(item)) - obj.attrs[key] = ret - else: - def _filler(): - obj.attrs[key] = self.__get_ref(value) - return _filler - - @docval({'name': 'parent', 'type': Group, 'doc': 'the parent HDF5 object'}, - {'name': 'builder', 'type': GroupBuilder, 'doc': 'the GroupBuilder to write'}, - returns='the Group that was created', rtype='Group') - def write_group(self, **kwargs): - - parent, builder = getargs('parent', 'builder', kwargs) - if builder.written: - group = parent[builder.name] - else: - group = parent.create_group(builder.name) - # write all groups - subgroups = builder.groups - if subgroups: - for subgroup_name, sub_builder in subgroups.items(): - # do not create an empty group without attributes or links - self.write_group(group, sub_builder) - # write all datasets - datasets = builder.datasets - if datasets: - for dset_name, sub_builder in datasets.items(): - self.write_dataset(group, sub_builder) - # write all links - links = builder.links - if links: - for link_name, sub_builder in links.items(): - self.write_link(group, sub_builder) - attributes = builder.attributes - self.set_attributes(group, attributes) - builder.written = True - return group - - def __get_path(self, builder): - curr = builder - names = list() - while curr is not None and curr.name != ROOT_NAME: - names.append(curr.name) - curr = curr.parent - delim = "/" - path = "%s%s" % (delim, delim.join(reversed(names))) - return path - - @docval({'name': 'parent', 'type': Group, 'doc': 'the parent HDF5 object'}, - {'name': 'builder', 'type': LinkBuilder, 'doc': 'the LinkBuilder to write'}, - returns='the Link that was created', rtype='Link') - def write_link(self, **kwargs): - parent, builder = getargs('parent', 'builder', kwargs) - if builder.written: - return None - name = builder.name - target_builder = builder.builder - path = self.__get_path(target_builder) - # source will indicate target_builder's location - if parent.file.filename == target_builder.source: - link_obj = SoftLink(path) - elif target_builder.source is not None: - link_obj = ExternalLink(target_builder.source, path) - else: - msg = 'cannot create external link to %s' % path - raise ValueError(msg) - parent[name] = link_obj - builder.written = True - return link_obj - - @docval({'name': 'parent', 'type': Group, 'doc': 'the parent HDF5 object'}, # noqa - {'name': 'builder', 'type': DatasetBuilder, 'doc': 'the DatasetBuilder to write'}, - {'name': 'link_data', 'type': bool, - 'doc': 'If not specified otherwise link (True) or copy (False) HDF5 Datasets', 'default': True}, - returns='the Dataset that was created', rtype=Dataset) - def write_dataset(self, **kwargs): - """ Write a dataset to HDF5 - - The function uses other dataset-dependent write functions, e.g, - __scalar_fill__, __list_fill__ and __chunked_iter_fill__ to write the data. - """ - parent, builder, link_data = getargs('parent', 'builder', 'link_data', kwargs) - if builder.written: - return None - name = builder.name - data = builder.data - options = dict() # dict with additional - if isinstance(data, H5DataIO): - options['io_settings'] = data.io_settings - link_data = data.link_data - data = data.data - else: - options['io_settings'] = {} - attributes = builder.attributes - options['dtype'] = builder.dtype - dset = None - link = None - - # The user provided an existing h5py dataset as input and asked to create a link to the dataset - if isinstance(data, Dataset): - # Create a Soft/External link to the dataset - if link_data: - data_filename = os.path.abspath(data.file.filename) - parent_filename = os.path.abspath(parent.file.filename) - if data_filename != parent_filename: - link = ExternalLink(os.path.relpath(data_filename, os.path.dirname(parent_filename)), data.name) - else: - link = SoftLink(data.name) - parent[name] = link - # Copy the dataset - else: - parent.copy(source=data, - dest=parent, - name=name, - expand_soft=False, - expand_external=False, - expand_refs=False, - without_attrs=True) - dset = parent[name] - # Write a compound dataset, i.e, a dataset with compound data type - elif isinstance(options['dtype'], list): - # do some stuff to figure out what data is a reference - refs = list() - for i, dts in enumerate(options['dtype']): - if self.__is_ref(dts): - refs.append(i) - # If one ore more of the parts of the compound data type are references then we need to deal with those - if len(refs) > 0: - try: - _dtype = self.__resolve_dtype__(options['dtype'], data) - except Exception as exc: - msg = 'cannot add %s to %s - could not determine type' % (name, parent.name) # noqa: F821 - raise_from(Exception(msg), exc) - dset = parent.require_dataset(name, shape=(len(data),), dtype=_dtype, **options['io_settings']) - builder.written = True - - @self.__queue_ref - def _filler(): - ret = list() - for item in data: - new_item = list(item) - for i in refs: - new_item[i] = self.__get_ref(item[i]) - ret.append(tuple(new_item)) - dset = parent[name] - dset[:] = ret - self.set_attributes(dset, attributes) - return - # If the compound data type contains only regular data (i.e., no references) then we can write it as usual - else: - dset = self.__list_fill__(parent, name, data, options) - # Write a dataset containing references, i.e., a region or object reference. - # NOTE: we can ignore options['io_settings'] for scalar data - elif self.__is_ref(options['dtype']): - _dtype = self.__dtypes.get(options['dtype']) - # Write a scalar data region reference dataset - if isinstance(data, RegionBuilder): - dset = parent.require_dataset(name, shape=(), dtype=_dtype) - builder.written = True - - @self.__queue_ref - def _filler(): - ref = self.__get_ref(data.builder, data.region) - dset = parent[name] - dset[()] = ref - self.set_attributes(dset, attributes) - # Write a scalar object reference dataset - elif isinstance(data, ReferenceBuilder): - dset = parent.require_dataset(name, dtype=_dtype, shape=()) - builder.written = True - - @self.__queue_ref - def _filler(): - ref = self.__get_ref(data.builder) - dset = parent[name] - dset[()] = ref - self.set_attributes(dset, attributes) - # Write an array dataset of references - else: - # Write a array of region references - if options['dtype'] == 'region': - dset = parent.require_dataset(name, dtype=_dtype, shape=(len(data),), **options['io_settings']) - builder.written = True - - @self.__queue_ref - def _filler(): - refs = list() - for item in data: - refs.append(self.__get_ref(item.builder, item.region)) - dset = parent[name] - dset[()] = refs - self.set_attributes(dset, attributes) - # Write array of object references - else: - dset = parent.require_dataset(name, shape=(len(data),), dtype=_dtype, ** options['io_settings']) - builder.written = True - - @self.__queue_ref - def _filler(): - refs = list() - for item in data: - refs.append(self.__get_ref(item)) - dset = parent[name] - dset[()] = refs - self.set_attributes(dset, attributes) - return - # write a "regular" dataset - else: - # Write a scalar dataset containing a single string - if isinstance(data, (text_type, binary_type)): - dset = self.__scalar_fill__(parent, name, data, options) - # Iterative write of a data chunk iterator - elif isinstance(data, AbstractDataChunkIterator): - dset = self.__chunked_iter_fill__(parent, name, data, options) - # Write a regular in memory array (e.g., numpy array, list etc.) - elif hasattr(data, '__len__'): - dset = self.__list_fill__(parent, name, data, options) - # Write a regular scalar dataset - else: - dset = self.__scalar_fill__(parent, name, data, options) - # Create the attributes on the dataset only if we are the primary and not just a Soft/External link - if link is None: - self.set_attributes(dset, attributes) - # Validate the attributes on the linked dataset - elif len(attributes) > 0: - pass - builder.written = True - return - - @classmethod - def __selection_max_bounds__(cls, selection): - """Determine the bounds of a numpy selection index tuple""" - if isinstance(selection, int): - return selection+1 - elif isinstance(selection, slice): - return selection.stop - elif isinstance(selection, list) or isinstance(selection, np.ndarray): - return np.nonzero(selection)[0][-1]+1 - elif isinstance(selection, tuple): - return tuple([cls.__selection_max_bounds__(i) for i in selection]) - - @classmethod - def __scalar_fill__(cls, parent, name, data, options=None): - dtype = None - io_settings = {} - if options is not None: - dtype = options.get('dtype') - io_settings = options.get('io_settings') - if not isinstance(dtype, type): - try: - dtype = cls.__resolve_dtype__(dtype, data) - except Exception as exc: - msg = 'cannot add %s to %s - could not determine type' % (name, parent.name) # noqa: F821 - raise_from(Exception(msg), exc) - try: - dset = parent.create_dataset(name, data=data, shape=None, dtype=dtype, **io_settings) - except Exception as exc: - msg = "Could not create scalar dataset %s in %s" % (name, parent.name) - raise_from(Exception(msg), exc) - return dset - - @classmethod - def __chunked_iter_fill__(cls, parent, name, data, options=None): - """ - Write data to a dataset one-chunk-at-a-time based on the given DataChunkIterator - - :param parent: The parent object to which the dataset should be added - :type parent: h5py.Group, h5py.File - :param name: The name of the dataset - :type name: str - :param data: The data to be written. - :type data: DataChunkIterator - :param options: Dict with options for creating a dataset. available options are 'dtype' and 'io_settings' - :type data: dict - - """ - io_settings = {} - if options is not None: - if 'io_settings' in options: - io_settings = options.get('io_settings') - # Define the chunking options if the user has not set them explicitly. We need chunking for the iterative write. - if 'chunks' not in io_settings: - recommended_chunks = data.recommended_chunk_shape() - io_settings['chunks'] = True if recommended_chunks is None else recommended_chunks - # Define the shape of the data if not provided by the user - if 'shape' not in io_settings: - io_settings['shape'] = data.recommended_data_shape() - # Define the maxshape of the data if not provided by the user - if 'maxshape' not in io_settings: - io_settings['maxshape'] = data.maxshape - if 'dtype' not in io_settings: - io_settings['dtype'] = data.dtype - try: - dset = parent.create_dataset(name, **io_settings) - except Exception as exc: - raise_from(Exception("Could not create dataset %s in %s" % (name, parent.name)), exc) - for chunk_i in data: - # Determine the minimum array dimensions to fit the chunk selection - max_bounds = cls.__selection_max_bounds__(chunk_i.selection) - if not hasattr(max_bounds, '__len__'): - max_bounds = (max_bounds,) - # Determine if we need to expand any of the data dimensions - expand_dims = [i for i, v in enumerate(max_bounds) if v is not None and v > dset.shape[i]] - # Expand the dataset if needed - if len(expand_dims) > 0: - new_shape = np.asarray(dset.shape) - new_shape[expand_dims] = np.asarray(max_bounds)[expand_dims] - dset.resize(new_shape) - # Process and write the data - dset[chunk_i.selection] = chunk_i.data - return dset - - @classmethod - def __list_fill__(cls, parent, name, data, options=None): - # define the io settings and data type if necessary - io_settings = {} - dtype = None - if options is not None: - dtype = options.get('dtype') - io_settings = options.get('io_settings') - if not isinstance(dtype, type): - try: - dtype = cls.__resolve_dtype__(dtype, data) - except Exception as exc: - msg = 'cannot add %s to %s - could not determine type' % (name, parent.name) # noqa: F821 - raise_from(Exception(msg), exc) - # define the data shape - if 'shape' in io_settings: - data_shape = io_settings.pop('shape') - elif isinstance(dtype, np.dtype): - data_shape = (len(data),) - else: - data_shape = get_shape(data) - # Create the dataset - try: - dset = parent.create_dataset(name, shape=data_shape, dtype=dtype, **io_settings) - except Exception as exc: - msg = "Could not create dataset %s in %s with shape %s, dtype %s, and iosettings %s. %s" % \ - (name, parent.name, str(data_shape), str(dtype), str(**io_settings), str(exc)) - raise_from(Exception(msg), exc) - # Write the data - if len(data) > dset.shape[0]: - new_shape = list(dset.shape) - new_shape[0] = len(data) - dset.resize(new_shape) - try: - dset[:] = data - except Exception as e: - raise e - return dset - - @docval({'name': 'container', 'type': (Builder, Container, ReferenceBuilder), 'doc': 'the object to reference'}, - {'name': 'region', 'type': (slice, list, tuple), 'doc': 'the region reference indexing object', - 'default': None}, - returns='the reference', rtype=Reference) - def __get_ref(self, **kwargs): - container, region = getargs('container', 'region', kwargs) - if isinstance(container, Builder): - if isinstance(container, LinkBuilder): - builder = container.target_builder - else: - builder = container - elif isinstance(container, ReferenceBuilder): - builder = container.builder - else: - builder = self.manager.build(container) - path = self.__get_path(builder) - if isinstance(container, RegionBuilder): - region = container.region - if region is not None: - dset = self.__file[path] - if not isinstance(dset, Dataset): - raise ValueError('cannot create region reference without Dataset') - return self.__file[path].regionref[region] - else: - return self.__file[path].ref - - def __is_ref(self, dtype): - if isinstance(dtype, DtypeSpec): - return self.__is_ref(dtype.dtype) - elif isinstance(dtype, RefSpec): - return True - else: - return dtype == DatasetBuilder.OBJECT_REF_TYPE or dtype == DatasetBuilder.REGION_REF_TYPE - - def __queue_ref(self, func): - '''Set aside filling dset with references - - dest[sl] = func() - - Args: - dset: the h5py.Dataset that the references need to be added to - sl: the np.s_ (slice) object for indexing into dset - func: a function to call to return the chunk of data, with - references filled in - ''' - # TODO: come up with more intelligent way of - # queueing reference resolution, based on reference - # dependency - self.__ref_queue.append(func) - - def __rec_get_ref(self, l): - ret = list() - for elem in l: - if isinstance(elem, (list, tuple)): - ret.append(self.__rec_get_ref(elem)) - elif isinstance(elem, (Builder, Container)): - ret.append(self.__get_ref(elem)) - else: - ret.append(elem) - return ret diff --git a/src/pynwb/form/backends/io.py b/src/pynwb/form/backends/io.py deleted file mode 100644 index 2583c3975..000000000 --- a/src/pynwb/form/backends/io.py +++ /dev/null @@ -1,68 +0,0 @@ -from abc import ABCMeta, abstractmethod -from ..build import BuildManager -from ..build import GroupBuilder -from ..utils import docval, getargs, popargs -from ..container import Container -from six import with_metaclass - - -class FORMIO(with_metaclass(ABCMeta, object)): - @docval({'name': 'manager', 'type': BuildManager, - 'doc': 'the BuildManager to use for I/O', 'default': None}, - {"name": "source", "type": str, - "doc": "the source of container being built i.e. file path", 'default': None}) - def __init__(self, **kwargs): - self.__manager = getargs('manager', kwargs) - self.__built = dict() - self.__source = getargs('source', kwargs) - self.open() - - @property - def manager(self): - '''The BuildManager this FORMIO is using''' - return self.__manager - - @property - def source(self): - '''The source of the container being read/written i.e. file path''' - return self.__source - - @docval(returns='the Container object that was read in', rtype=Container) - def read(self, **kwargs): - f_builder = self.read_builder() - container = self.__manager.construct(f_builder) - return container - - @docval({'name': 'container', 'type': Container, 'doc': 'the Container object to write'}) - def write(self, **kwargs): - container = popargs('container', kwargs) - f_builder = self.__manager.build(container, source=self.__source) - self.write_builder(f_builder, **kwargs) - - @abstractmethod - @docval(returns='a GroupBuilder representing the read data', rtype='GroupBuilder') - def read_builder(self): - ''' Read data and return the GroupBuilder representing ''' - pass - - @abstractmethod - @docval({'name': 'builder', 'type': GroupBuilder, 'doc': 'the GroupBuilder object representing the Container'}) - def write_builder(self, **kwargs): - ''' Write a GroupBuilder representing an Container object ''' - pass - - @abstractmethod - def open(self): - ''' Open this FORMIO object for writing of the builder ''' - pass - - @abstractmethod - def close(self): - ''' Close this FORMIO object to further reading/writing''' - pass - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() diff --git a/src/pynwb/form/build/__init__.py b/src/pynwb/form/build/__init__.py deleted file mode 100644 index 84310dc28..000000000 --- a/src/pynwb/form/build/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# flake8: noqa: F401 -from .builders import Builder -from .builders import GroupBuilder -from .builders import DatasetBuilder -from .builders import ReferenceBuilder -from .builders import RegionBuilder -from .builders import LinkBuilder - -from .map import ObjectMapper -from .map import BuildManager -from .map import TypeMap diff --git a/src/pynwb/form/build/builders.py b/src/pynwb/form/build/builders.py deleted file mode 100644 index 1475dbeaa..000000000 --- a/src/pynwb/form/build/builders.py +++ /dev/null @@ -1,536 +0,0 @@ -import numpy as np -from h5py import RegionReference -import copy as _copy -import itertools as _itertools -import posixpath as _posixpath -from abc import ABCMeta -import warnings -from collections import Iterable - -from ..utils import docval, getargs, popargs, call_docval_func, fmt_docval_args -from six import with_metaclass - - -class Builder(with_metaclass(ABCMeta, dict)): - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'}, - {'name': 'parent', 'type': 'Builder', 'doc': 'the parent builder of this Builder', 'default': None}, - {'name': 'source', 'type': str, - 'doc': 'the source of the data in this builder e.g. file name', 'default': None}) - def __init__(self, **kwargs): - name, parent, source = getargs('name', 'parent', 'source', kwargs) - super(Builder, self).__init__() - self.__name = name - self.__parent = parent - if source is not None: - self.__source = source - elif parent is not None: - self.__source = parent.source - else: - self.__source = None - self.__written = False - - @property - def path(self): - """ - Get the path of this Builder - """ - s = list() - c = self - while c is not None: - s.append(c.name) - c = c.parent - return "/".join(s[::-1]) - - @property - def written(self): - ''' The source of this Builder ''' - return self.__written - - @written.setter - def written(self, s): - if self.__written and not s: - raise ValueError("cannot change written to not written") - self.__written = s - - @property - def name(self): - ''' The name of this Builder ''' - return self.__name - - @property - def source(self): - ''' The source of this Builder ''' - return self.__source - - @source.setter - def source(self, s): - if self.__source is None: - self.__source = s - else: - raise ValueError('Cannot reset source once it is specified') - - @property - def parent(self): - ''' The parent Builder of this Builder ''' - return self.__parent - - @parent.setter - def parent(self, p): - if self.__parent is None: - self.__parent = p - if self.__source is None: - self.source = p.source - else: - raise ValueError('Cannot reset parent once it is specified') - - -class BaseBuilder(Builder): - __attribute = 'attributes' - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'}, - {'name': 'attributes', 'type': dict, 'doc': 'a dictionary of attributes to create in this group', - 'default': dict()}, - {'name': 'parent', 'type': 'GroupBuilder', 'doc': 'the parent builder of this Builder', 'default': None}, - {'name': 'source', 'type': str, - 'doc': 'the source of the data represented in this Builder', 'default': None}) - def __init__(self, **kwargs): - name, attributes, parent, source = getargs('name', 'attributes', 'parent', 'source', kwargs) - super(BaseBuilder, self).__init__(name, parent, source) - super(BaseBuilder, self).__setitem__(BaseBuilder.__attribute, dict()) - for name, val in attributes.items(): - self.set_attribute(name, val) - - @property - def attributes(self): - ''' The attributes stored in this Builder object ''' - return super(BaseBuilder, self).__getitem__(BaseBuilder.__attribute) - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute'}, - {'name': 'value', 'type': None, 'doc': 'the attribute value'}) - def set_attribute(self, **kwargs): - ''' Set an attribute for this group. ''' - name, value = getargs('name', 'value', kwargs) - super(BaseBuilder, self).__getitem__(BaseBuilder.__attribute)[name] = value - # self.obj_type[name] = BaseBuilder.__attribute - - @docval({'name': 'builder', 'type': 'BaseBuilder', 'doc': 'the BaseBuilder to merge attributes from '}) - def deep_update(self, **kwargs): - ''' Merge attributes from the given BaseBuilder into this builder ''' - builder = kwargs['builder'] - # merge attributes - for name, value in super(BaseBuilder, builder).__getitem__(BaseBuilder.__attribute).items(): - self.set_attribute(name, value) - - -class GroupBuilder(BaseBuilder): - __link = 'links' - __group = 'groups' - __dataset = 'datasets' - __attribute = 'attributes' - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'}, - {'name': 'groups', 'type': (dict, list), 'doc': 'a dictionary of subgroups to create in this group', - 'default': dict()}, - {'name': 'datasets', 'type': (dict, list), 'doc': 'a dictionary of datasets to create in this group', - 'default': dict()}, - {'name': 'attributes', 'type': dict, 'doc': 'a dictionary of attributes to create in this group', - 'default': dict()}, - {'name': 'links', 'type': (dict, list), 'doc': 'a dictionary of links to create in this group', - 'default': dict()}, - {'name': 'parent', 'type': 'GroupBuilder', 'doc': 'the parent builder of this Builder', 'default': None}, - {'name': 'source', 'type': str, - 'doc': 'the source of the data represented in this Builder', 'default': None}) - def __init__(self, **kwargs): - ''' - Create a GroupBuilder object - ''' - name, groups, datasets, links, attributes, parent, source = getargs( - 'name', 'groups', 'datasets', 'links', 'attributes', 'parent', 'source', kwargs) - groups = self.__to_list(groups) - datasets = self.__to_list(datasets) - links = self.__to_list(links) - self.obj_type = dict() - super(GroupBuilder, self).__init__(name, attributes, parent, source) - super(GroupBuilder, self).__setitem__(GroupBuilder.__group, dict()) - super(GroupBuilder, self).__setitem__(GroupBuilder.__dataset, dict()) - super(GroupBuilder, self).__setitem__(GroupBuilder.__link, dict()) - self.__name = name - for group in groups: - self.set_group(group) - for dataset in datasets: - if not (dataset is None): - self.set_dataset(dataset) - for link in links: - self.set_link(link) - - def __to_list(self, d): - if isinstance(d, dict): - return list(d.values()) - return d - - @property - def source(self): - ''' The source of this Builder ''' - return super(GroupBuilder, self).source - - @source.setter - def source(self, s): - ''' - A recursive setter to set all subgroups/datasets/links - source when this source is set - ''' - super(GroupBuilder, self.__class__).source.fset(self, s) - for g in self.groups.values(): - if g.source is None: - g.source = s - for d in self.datasets.values(): - if d.source is None: - d.source = s - for l in self.links.values(): - if l.source is None: - l.source = s - - @property - def groups(self): - ''' The subgroups contained in this GroupBuilder ''' - return super(GroupBuilder, self).__getitem__(GroupBuilder.__group) - - @property - def datasets(self): - ''' The datasets contained in this GroupBuilder ''' - return super(GroupBuilder, self).__getitem__(GroupBuilder.__dataset) - - @property - def links(self): - ''' The datasets contained in this GroupBuilder ''' - return super(GroupBuilder, self).__getitem__(GroupBuilder.__link) - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute'}, - {'name': 'value', 'type': None, 'doc': 'the attribute value'}) - def set_attribute(self, **kwargs): - ''' Set an attribute for this group ''' - name, value = getargs('name', 'value', kwargs) - super(GroupBuilder, self).set_attribute(name, value) - self.obj_type[name] = GroupBuilder.__attribute - - @docval({'name': 'builder', 'type': 'Builder', 'doc': 'the Builder to add to this GroupBuilder'}) - def set_builder(self, **kwargs): - ''' - Add an existing builder to this this GroupBuilder - ''' - builder = getargs('builder', kwargs) - if isinstance(builder, LinkBuilder): - self.__set_builder(builder, GroupBuilder.__link) - elif isinstance(builder, GroupBuilder): - self.__set_builder(builder, GroupBuilder.__dataset) - elif isinstance(builder, DatasetBuilder): - self.__set_builder(builder, GroupBuilder.__dataset) - else: - raise ValueError("Got unexpected builder type: %s" % type(builder)) - - def __set_builder(self, builder, obj_type): - name = builder.name - if name in self.obj_type: - if self.obj_type[name] != obj_type: - if name == 'comments': - # LEGACY: Support legacy files where "comments" exists as both an attribute and as dataset - # in some groups. - # To allow read to get past this special case, this will skip the issue. - warnings.warn("'%s' already exists as %s; skipping..." % (name, self.obj_type[name])) - else: - raise KeyError("'%s' already exists as %s in %s, cannot set as %s" % - (name, self.obj_type[name], self.name, obj_type)) - super(GroupBuilder, self).__getitem__(obj_type)[name] = builder - self.obj_type[name] = obj_type - if builder.parent is None: - builder.parent = self - - @docval({'name': 'name', 'type': str, 'doc': 'the name of this dataset'}, - {'name': 'data', 'type': ('array_data', 'scalar_data', 'data', 'DatasetBuilder', Iterable), - 'doc': 'a dictionary of datasets to create in this dataset', 'default': None}, - {'name': 'dtype', 'type': (type, np.dtype, str, list), - 'doc': 'the datatype of this dataset', 'default': None}, - {'name': 'attributes', 'type': dict, - 'doc': 'a dictionary of attributes to create in this dataset', 'default': dict()}, - {'name': 'maxshape', 'type': (int, tuple), - 'doc': 'the shape of this dataset. Use None for scalars', 'default': None}, - {'name': 'chunks', 'type': bool, 'doc': 'whether or not to chunk this dataset', 'default': False}, - returns='the DatasetBuilder object for the dataset', rtype='DatasetBuilder') - def add_dataset(self, **kwargs): - ''' Create a dataset and add it to this group ''' - pargs, pkwargs = fmt_docval_args(DatasetBuilder.__init__, kwargs) - pkwargs['parent'] = self - pkwargs['source'] = self.source - builder = DatasetBuilder(*pargs, **pkwargs) - self.set_dataset(builder) - return builder - - @docval({'name': 'builder', 'type': 'DatasetBuilder', 'doc': 'the DatasetBuilder that represents this dataset'}) - def set_dataset(self, **kwargs): - ''' Add a dataset to this group ''' - builder = getargs('builder', kwargs) - self.__set_builder(builder, GroupBuilder.__dataset) - - @docval({'name': 'name', 'type': str, 'doc': 'the name of this subgroup'}, - {'name': 'groups', 'type': dict, - 'doc': 'a dictionary of subgroups to create in this subgroup', 'default': dict()}, - {'name': 'datasets', 'type': dict, - 'doc': 'a dictionary of datasets to create in this subgroup', 'default': dict()}, - {'name': 'attributes', 'type': dict, - 'doc': 'a dictionary of attributes to create in this subgroup', 'default': dict()}, - {'name': 'links', 'type': dict, - 'doc': 'a dictionary of links to create in this subgroup', 'default': dict()}, - returns='the GroupBuilder object for the subgroup', rtype='GroupBuilder') - def add_group(self, **kwargs): - ''' Add a subgroup with the given data to this group ''' - name = kwargs.pop('name') - builder = GroupBuilder(name, parent=self, **kwargs) - self.set_group(builder) - return builder - - @docval({'name': 'builder', 'type': 'GroupBuilder', 'doc': 'the GroupBuilder that represents this subgroup'}) - def set_group(self, **kwargs): - ''' Add a subgroup to this group ''' - builder = getargs('builder', kwargs) - self.__set_builder(builder, GroupBuilder.__group) - - @docval({'name': 'target', 'type': ('GroupBuilder', 'DatasetBuilder'), 'doc': 'the target Builder'}, - {'name': 'name', 'type': str, 'doc': 'the name of this link', 'default': None}, - returns='the builder object for the soft link', rtype='LinkBuilder') - def add_link(self, **kwargs): - ''' Create a soft link and add it to this group ''' - name, target = getargs('name', 'target', kwargs) - builder = LinkBuilder(target, name, self) - self.set_link(builder) - return builder - - @docval({'name': 'builder', 'type': 'LinkBuilder', 'doc': 'the LinkBuilder that represents this link'}) - def set_link(self, **kwargs): - ''' Add a link to this group ''' - builder = getargs('builder', kwargs) - self.__set_builder(builder, GroupBuilder.__link) - - # TODO: write unittests for this method - def deep_update(self, builder): - ''' Recursively update subgroups in this group ''' - super(GroupBuilder, self).deep_update(builder) - # merge subgroups - groups = super(GroupBuilder, builder).__getitem__(GroupBuilder.__group) - self_groups = super(GroupBuilder, self).__getitem__(GroupBuilder.__group) - for name, subgroup in groups.items(): - if name in self_groups: - self_groups[name].deep_update(subgroup) - else: - self.set_group(subgroup) - # merge datasets - datasets = super(GroupBuilder, builder).__getitem__(GroupBuilder.__dataset) - self_datasets = super(GroupBuilder, self).__getitem__(GroupBuilder.__dataset) - for name, dataset in datasets.items(): - # self.add_dataset(name, dataset) - if name in self_datasets: - self_datasets[name].deep_update(dataset) - # super().__getitem__(GroupBuilder.__dataset)[name] = dataset - else: - self.set_dataset(dataset) - # merge links - for name, link in super(GroupBuilder, builder).__getitem__(GroupBuilder.__link).items(): - self.set_link(link) - - def is_empty(self): - '''Returns true if there are no datasets, attributes, links or - subgroups that contain datasets, attributes or links. False otherwise. - ''' - if (len(super(GroupBuilder, self).__getitem__(GroupBuilder.__dataset)) or - len(super(GroupBuilder, self).__getitem__(GroupBuilder.__attribute)) or - len(super(GroupBuilder, self).__getitem__(GroupBuilder.__link))): # noqa: E129 - return False - elif len(super(GroupBuilder, self).__getitem__(GroupBuilder.__group)): - return all(g.is_empty() for g in super(GroupBuilder, self).__getitem__(GroupBuilder.__group).values()) - else: - return True - - def __getitem__(self, key): - '''Like dict.__getitem__, but looks in groups, - datasets, attributes, and links sub-dictionaries. - ''' - try: - key_ar = _posixpath.normpath(key).split('/') - return self.__get_rec(key_ar) - except KeyError: - raise KeyError(key) - - def get(self, key, default=None): - '''Like dict.get, but looks in groups, - datasets, attributes, and links sub-dictionaries. - ''' - try: - key_ar = _posixpath.normpath(key).split('/') - return self.__get_rec(key_ar) - except KeyError: - return default - - def __get_rec(self, key_ar): - # recursive helper for __getitem__ - if len(key_ar) == 1: - return super(GroupBuilder, self).__getitem__(self.obj_type[key_ar[0]])[key_ar[0]] - else: - if key_ar[0] in super(GroupBuilder, self).__getitem__(GroupBuilder.__group): - return super(GroupBuilder, self).__getitem__(GroupBuilder.__group)[key_ar[0]].__get_rec(key_ar[1:]) - raise KeyError(key_ar[0]) - - def __setitem__(self, args, val): - raise NotImplementedError('__setitem__') - - def __contains__(self, item): - return self.obj_type.__contains__(item) - - def items(self): - '''Like dict.items, but iterates over key-value pairs in groups, - datasets, attributes, and links sub-dictionaries. - ''' - return _itertools.chain(super(GroupBuilder, self).__getitem__(GroupBuilder.__group).items(), - super(GroupBuilder, self).__getitem__(GroupBuilder.__dataset).items(), - super(GroupBuilder, self).__getitem__(GroupBuilder.__attribute).items(), - super(GroupBuilder, self).__getitem__(GroupBuilder.__link).items()) - - def keys(self): - '''Like dict.keys, but iterates over keys in groups, datasets, - attributes, and links sub-dictionaries. - ''' - return _itertools.chain(super(GroupBuilder, self).__getitem__(GroupBuilder.__group).keys(), - super(GroupBuilder, self).__getitem__(GroupBuilder.__dataset).keys(), - super(GroupBuilder, self).__getitem__(GroupBuilder.__attribute).keys(), - super(GroupBuilder, self).__getitem__(GroupBuilder.__link).keys()) - - def values(self): - '''Like dict.values, but iterates over values in groups, datasets, - attributes, and links sub-dictionaries. - ''' - return _itertools.chain(super(GroupBuilder, self).__getitem__(GroupBuilder.__group).values(), - super(GroupBuilder, self).__getitem__(GroupBuilder.__dataset).values(), - super(GroupBuilder, self).__getitem__(GroupBuilder.__attribute).values(), - super(GroupBuilder, self).__getitem__(GroupBuilder.__link).values()) - - -class DatasetBuilder(BaseBuilder): - OBJECT_REF_TYPE = 'object' - REGION_REF_TYPE = 'region' - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the dataset'}, - {'name': 'data', 'type': ('array_data', 'scalar_data', 'data', 'DatasetBuilder', 'RegionBuilder', Iterable), - 'doc': 'the data in this dataset', 'default': None}, - {'name': 'dtype', 'type': (type, np.dtype, str, list), - 'doc': 'the datatype of this dataset', 'default': None}, - {'name': 'attributes', 'type': dict, - 'doc': 'a dictionary of attributes to create in this dataset', 'default': dict()}, - {'name': 'maxshape', 'type': (int, tuple), - 'doc': 'the shape of this dataset. Use None for scalars', 'default': None}, - {'name': 'chunks', 'type': bool, 'doc': 'whether or not to chunk this dataset', 'default': False}, - {'name': 'parent', 'type': GroupBuilder, 'doc': 'the parent builder of this Builder', 'default': None}, - {'name': 'source', 'type': str, 'doc': 'the source of the data in this builder', 'default': None}) - def __init__(self, **kwargs): - ''' Create a Builder object for a dataset ''' - name, data, dtype, attributes, maxshape, chunks, parent, source = getargs( - 'name', 'data', 'dtype', 'attributes', 'maxshape', 'chunks', 'parent', 'source', kwargs) - super(DatasetBuilder, self).__init__(name, attributes, parent, source) - self['data'] = data - self['attributes'] = _copy.copy(attributes) - self.__chunks = chunks - self.__maxshape = maxshape - if isinstance(data, BaseBuilder): - if dtype is None: - dtype = self.OBJECT_REF_TYPE - self.__dtype = dtype - self.__name = name - - @property - def data(self): - ''' The data stored in the dataset represented by this builder ''' - return self['data'] - - @data.setter - def data(self, val): - if self['data'] is not None: - raise AttributeError("'data' already set") - self['data'] = val - - @property - def chunks(self): - ''' Whether or not this dataset is chunked ''' - return self.__chunks - - @property - def maxshape(self): - ''' The max shape of this object ''' - return self.__maxshape - - @property - def dtype(self): - ''' The data type of this object ''' - return self.__dtype - - @dtype.setter - def dtype(self, val): - ''' The data type of this object ''' - if self.__dtype is None: - self.__dtype = val - else: - raise AttributeError("cannot overwrite dtype") - - @docval({'name': 'dataset', 'type': 'DatasetBuilder', - 'doc': 'the DatasetBuilder to merge into this DatasetBuilder'}) - def deep_update(self, **kwargs): - '''Merge data and attributes from given DatasetBuilder into this DatasetBuilder''' - dataset = getargs('dataset', kwargs) - if dataset.data: - self['data'] = dataset.data # TODO: figure out if we want to add a check for overwrite - self['attributes'].update(dataset.attributes) - - -class LinkBuilder(Builder): - - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), 'doc': 'the target of this link'}, - {'name': 'name', 'type': str, 'doc': 'the name of the dataset', 'default': None}, - {'name': 'parent', 'type': GroupBuilder, 'doc': 'the parent builder of this Builder', 'default': None}, - {'name': 'source', 'type': str, 'doc': 'the source of the data in this builder', 'default': None}) - def __init__(self, **kwargs): - name, builder, parent, source = getargs('name', 'builder', 'parent', 'source', kwargs) - if name is None: - name = builder.name - super(LinkBuilder, self).__init__(name, parent, source) - self['builder'] = builder - - @property - def builder(self): - ''' The target builder object ''' - return self['builder'] - - -class ReferenceBuilder(dict): - - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), 'doc': 'the Dataset this region applies to'}) - def __init__(self, **kwargs): - builder = getargs('builder', kwargs) - self['builder'] = builder - - @property - def builder(self): - ''' The target builder object ''' - return self['builder'] - - -class RegionBuilder(ReferenceBuilder): - - @docval({'name': 'region', 'type': (slice, tuple, list, RegionReference), - 'doc': 'the region i.e. slice or indices into the target Dataset'}, - {'name': 'builder', 'type': DatasetBuilder, 'doc': 'the Dataset this region applies to'}) - def __init__(self, **kwargs): - region = popargs('region', kwargs) - call_docval_func(super(RegionBuilder, self).__init__, kwargs) - self['region'] = region - - @property - def region(self): - ''' The target builder object ''' - return self['region'] diff --git a/src/pynwb/form/build/map.py b/src/pynwb/form/build/map.py deleted file mode 100644 index 815b74b72..000000000 --- a/src/pynwb/form/build/map.py +++ /dev/null @@ -1,1576 +0,0 @@ -from __future__ import absolute_import -import re -import numpy as np -import warnings -from collections import OrderedDict -from copy import copy -from datetime import datetime -from six import with_metaclass, raise_from, text_type, binary_type, integer_types - -from ..utils import docval, getargs, ExtenderMeta, get_docval, fmt_docval_args, call_docval_func -from ..container import Container, Data, DataRegion -from ..spec import Spec, AttributeSpec, DatasetSpec, GroupSpec, LinkSpec, NAME_WILDCARD, NamespaceCatalog, RefSpec,\ - SpecReader -from ..data_utils import DataIO, AbstractDataChunkIterator -from ..spec.spec import BaseStorageSpec -from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, ReferenceBuilder, RegionBuilder, BaseBuilder -from .warnings import OrphanContainerWarning, MissingRequiredWarning - - -class Proxy(object): - """ - A temporary object to represent a Container. This gets used when resolving the true location of a - Container's parent. - - Proxy objects allow simple bookkeeping of all potential parents a Container may have. - - This object is used by providing all the necessary information for describing the object. This object - gets passed around and candidates are accumulated. Upon calling resolve, all saved candidates are matched - against the information (provided to the constructor). The candidate that has an exact match is returned. - """ - - def __init__(self, manager, source, location, namespace, data_type): - self.__source = source - self.__location = location - self.__namespace = namespace - self.__data_type = data_type - self.__manager = manager - self.__candidates = list() - - @property - def source(self): - """The source of the object e.g. file source""" - return self.__source - - @property - def location(self): - """The location of the object. This can be thought of as a unique path""" - return self.__location - - @property - def namespace(self): - """The namespace from which the data_type of this Proxy came from""" - return self.__namespace - - @property - def data_type(self): - """The data_type of Container that should match this Proxy""" - return self.__data_type - - @docval({"name": "object", "type": (BaseBuilder, Container), "doc": "the container or builder to get a proxy for"}) - def matches(self, **kwargs): - obj = getargs('object', kwargs) - if not isinstance(obj, Proxy): - obj = self.__manager.get_proxy(obj) - return self == obj - - @docval({"name": "container", "type": Container, "doc": "the Container to add as a candidate match"}) - def add_candidate(self, **kwargs): - container = getargs('container', kwargs) - self.__candidates.append(container) - - def resolve(self, **kwargs): - for candidate in self.__candidates: - if self.matches(candidate): - return candidate - return None - - def __eq__(self, other): - return self.data_type == other.data_type and \ - self.location == other.location and \ - self.namespace == other.namespace and \ - self.source == other.source - - def __repr__(self): - ret = dict() - for key in ('source', 'location', 'namespace', 'data_type'): - ret[key] = getattr(self, key, None) - return str(ret) - - -class BuildManager(object): - """ - A class for managing builds of Containers - """ - - def __init__(self, type_map): - self.__builders = dict() - self.__containers = dict() - self.__type_map = type_map - - @property - def namespace_catalog(self): - return self.__type_map.namespace_catalog - - @property - def type_map(self): - return self.__type_map - - @docval({"name": "object", "type": (BaseBuilder, Container), "doc": "the container or builder to get a proxy for"}, - {"name": "source", "type": str, - "doc": "the source of container being built i.e. file path", 'default': None}) - def get_proxy(self, **kwargs): - obj = getargs('object', kwargs) - if isinstance(obj, BaseBuilder): - return self.__get_proxy_builder(obj) - elif isinstance(obj, Container): - return self.__get_proxy_container(obj) - - def __get_proxy_builder(self, builder): - dt = self.__type_map.get_builder_dt(builder) - ns = self.__type_map.get_builder_ns(builder) - stack = list() - tmp = builder - while tmp is not None: - stack.append(tmp.name) - tmp = self.__get_parent_dt_builder(tmp) - loc = "/".join(reversed(stack)) - return Proxy(self, builder.source, loc, ns, dt) - - def __get_proxy_container(self, container): - ns, dt = self.__type_map.get_container_ns_dt(container) - stack = list() - tmp = container - while tmp is not None: - if isinstance(tmp, Proxy): - stack.append(tmp.location) - break - else: - stack.append(tmp.name) - tmp = tmp.parent - loc = "/".join(reversed(stack)) - return Proxy(self, container.container_source, loc, ns, dt) - - @docval({"name": "container", "type": Container, "doc": "the container to convert to a Builder"}, - {"name": "source", "type": str, - "doc": "the source of container being built i.e. file path", 'default': None}) - def build(self, **kwargs): - """ Build the GroupBuilder for the given Container""" - container = getargs('container', kwargs) - container_id = self.__conthash__(container) - result = self.__builders.get(container_id) - source = getargs('source', kwargs) - if result is None: - if container.container_source is None: - container.container_source = source - else: - if container.container_source != source: - raise ValueError("Can't change container_source once set") - result = self.__type_map.build(container, self, source=source) - self.prebuilt(container, result) - elif container.modified: - if isinstance(result, GroupBuilder): - # TODO: if Datasets attributes are allowed to be modified, we need to - # figure out how to handle that starting here. - result = self.__type_map.build(container, self, builder=result, source=source) - return result - - @docval({"name": "container", "type": Container, "doc": "the Container to save as prebuilt"}, - {'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), - 'doc': 'the Builder representation of the given container'}) - def prebuilt(self, **kwargs): - ''' Save the Builder for a given Container for future use ''' - container, builder = getargs('container', 'builder', kwargs) - container_id = self.__conthash__(container) - self.__builders[container_id] = builder - builder_id = self.__bldrhash__(builder) - self.__containers[builder_id] = container - - def __conthash__(self, obj): - return id(obj) - - def __bldrhash__(self, obj): - return id(obj) - - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), - 'doc': 'the builder to construct the Container from'}) - def construct(self, **kwargs): - """ Construct the Container represented by the given builder """ - builder = getargs('builder', kwargs) - if isinstance(builder, LinkBuilder): - builder = builder.target - builder_id = self.__bldrhash__(builder) - result = self.__containers.get(builder_id) - if result is None: - result = self.__type_map.construct(builder, self) - parent_builder = self.__get_parent_dt_builder(builder) - if parent_builder is not None: - result.parent = self.__get_proxy_builder(parent_builder) - else: - # we are at the top of the hierarchy, - # so it must be time to resolve parents - self.__resolve_parents(result) - self.prebuilt(result, builder) - result.set_modified(False) - return result - - def __resolve_parents(self, container): - stack = [container] - while len(stack) > 0: - tmp = stack.pop() - if isinstance(tmp.parent, Proxy): - tmp.parent = tmp.parent.resolve() - for child in tmp.children: - stack.append(child) - - def __get_parent_dt_builder(self, builder): - ''' - Get the next builder above the given builder - that has a data_type - ''' - tmp = builder.parent - ret = None - while tmp is not None: - ret = tmp - dt = self.__type_map.get_builder_dt(tmp) - if dt is not None: - break - tmp = tmp.parent - return ret - - @docval({'name': 'builder', 'type': Builder, 'doc': 'the Builder to get the class object for'}) - def get_cls(self, **kwargs): - ''' Get the class object for the given Builder ''' - builder = getargs('builder', kwargs) - return self.__type_map.get_cls(builder) - - @docval({"name": "container", "type": Container, "doc": "the container to convert to a Builder"}, - returns='The name a Builder should be given when building this container', rtype=str) - def get_builder_name(self, **kwargs): - ''' Get the name a Builder should be given ''' - container = getargs('container', kwargs) - return self.__type_map.get_builder_name(container) - - @docval({'name': 'spec', 'type': (DatasetSpec, GroupSpec), 'doc': 'the parent spec to search'}, - {'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), - 'doc': 'the builder to get the sub-specification for'}) - def get_subspec(self, **kwargs): - ''' - Get the specification from this spec that corresponds to the given builder - ''' - spec, builder = getargs('spec', 'builder', kwargs) - return self.__type_map.get_subspec(spec, builder) - - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), - 'doc': 'the builder to get the sub-specification for'}) - def get_builder_ns(self, **kwargs): - ''' - Get the namespace of a builder - ''' - builder = getargs('builder', kwargs) - return self.__type_map.get_builder_ns(builder) - - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), - 'doc': 'the builder to get the data_type for'}) - def get_builder_dt(self, **kwargs): - ''' - Get the data_type of a builder - ''' - builder = getargs('builder', kwargs) - return self.__type_map.get_builder_dt(builder) - - -_const_arg = '__constructor_arg' - - -@docval({'name': 'name', 'type': str, 'doc': 'the name of the constructor argument'}, - is_method=False) -def _constructor_arg(**kwargs): - '''Decorator to override the default mapping scheme for a given constructor argument. - - Decorate ObjectMapper methods with this function when extending ObjectMapper to override the default - scheme for mapping between Container and Builder objects. The decorated method should accept as its - first argument the Builder object that is being mapped. The method should return the value to be passed - to the target Container class constructor argument given by *name*. - ''' - name = getargs('name', kwargs) - - def _dec(func): - setattr(func, _const_arg, name) - return func - return _dec - - -_obj_attr = '__object_attr' - - -@docval({'name': 'name', 'type': str, 'doc': 'the name of the constructor argument'}, - is_method=False) -def _object_attr(**kwargs): - '''Decorator to override the default mapping scheme for a given object attribute. - - Decorate ObjectMapper methods with this function when extending ObjectMapper to override the default - scheme for mapping between Container and Builder objects. The decorated method should accept as its - first argument the Container object that is being mapped. The method should return the child Builder - object (or scalar if the object attribute corresponds to an AttributeSpec) that represents the - attribute given by *name*. - ''' - name = getargs('name', kwargs) - - def _dec(func): - setattr(func, _obj_attr, name) - return func - return _dec - - -def _unicode(s): - """ - A helper function for converting to Unicode - """ - if isinstance(s, text_type): - return s - elif isinstance(s, binary_type): - return s.decode('utf-8') - else: - raise ValueError("Expected unicode or ascii string, got %s" % type(s)) - - -def _ascii(s): - """ - A helper function for converting to ASCII - """ - if isinstance(s, text_type): - return s.encode('ascii', 'backslashreplace') - elif isinstance(s, binary_type): - return s - else: - raise ValueError("Expected unicode or ascii string, got %s" % type(s)) - - -class ObjectMapper(with_metaclass(ExtenderMeta, object)): - '''A class for mapping between Spec objects and Container attributes - - ''' - - __dtypes = { - "float": np.float32, - "float32": np.float32, - "double": np.float64, - "float64": np.float64, - "long": np.int64, - "int64": np.int64, - "uint64": np.uint64, - "int": np.int32, - "int32": np.int32, - "int16": np.int16, - "int8": np.int8, - "bool": np.bool_, - "text": _unicode, - "text": _unicode, - "utf": _unicode, - "utf8": _unicode, - "utf-8": _unicode, - "ascii": _ascii, - "str": _ascii, - "isodatetime": _ascii, - "uint32": np.uint32, - "uint16": np.uint16, - "uint8": np.uint8, - } - - @classmethod - def __resolve_dtype(cls, given, specified): - """ - Determine the dtype to use from the dtype of the given value and the specified dtype. - This amounts to determining the greater precision of the two arguments, but also - checks to make sure the same base dtype is being used. - """ - g = np.dtype(given) - s = np.dtype(specified) - if g.itemsize <= s.itemsize: - return s.type - else: - if g.name[:3] != s.name[:3]: # different types - if s.itemsize < 8: - msg = "expected %s, received %s - must supply %s or higher precision" % (s.name, g.name, s.name) - else: - msg = "expected %s, received %s - must supply %s" % (s.name, g.name, s.name) - raise ValueError(msg) - else: - return g.type - - @classmethod - def convert_dtype(cls, spec, value): - """ - Convert values to the specified dtype. For example, if a literal int - is passed in to a field that is specified as a unsigned integer, this function - will convert the Python int to a numpy unsigned int. - - :return: The function returns a tuple consisting of 1) the value, and 2) the data type. - The value is returned as the function may convert the input value to comply - with the dtype specified in the schema. - """ - if value is None: - dt = spec.dtype - if isinstance(dt, RefSpec): - dt = dt.reftype - return None, dt - if isinstance(value, DataIO): - return value, cls.convert_dtype(spec, value.data)[1] - if spec.dtype is None: - return value, None - if spec.dtype == 'numeric': - return value, None - if spec.dtype is not None and spec.dtype not in cls.__dtypes: - msg = "unrecognized dtype: %s -- cannot convert value" % spec.dtype - raise ValueError(msg) - ret = None - ret_dtype = None - spec_dtype = cls.__dtypes[spec.dtype] - if isinstance(value, np.ndarray): - if spec_dtype is _unicode: - ret = value.astype('U') - ret_dtype = "utf8" - elif spec_dtype is _ascii: - ret = value.astype('S') - ret_dtype = "ascii" - else: - dtype_func = cls.__resolve_dtype(value.dtype, spec_dtype) - ret = value.astype(dtype_func) - ret_dtype = ret.dtype.type - elif isinstance(value, (tuple, list)): - ret = list() - for elem in value: - tmp, tmp_dtype = cls.convert_dtype(spec, elem) - ret.append(tmp) - ret = type(value)(ret) - ret_dtype = tmp_dtype - else: - if spec_dtype in (_unicode, _ascii): - ret_dtype = 'ascii' - if spec_dtype == _unicode: - ret_dtype = 'utf8' - ret = spec_dtype(value) - else: - dtype_func = cls.__resolve_dtype(type(value), spec_dtype) - ret = dtype_func(value) - ret_dtype = type(ret) - return ret, ret_dtype - - _const_arg = '__constructor_arg' - - @staticmethod - @docval({'name': 'name', 'type': str, 'doc': 'the name of the constructor argument'}, - is_method=False) - def constructor_arg(**kwargs): - '''Decorator to override the default mapping scheme for a given constructor argument. - - Decorate ObjectMapper methods with this function when extending ObjectMapper to override the default - scheme for mapping between Container and Builder objects. The decorated method should accept as its - first argument the Builder object that is being mapped. The method should return the value to be passed - to the target Container class constructor argument given by *name*. - ''' - name = getargs('name', kwargs) - return _constructor_arg(name) - - _obj_attr = '__object_attr' - - @staticmethod - @docval({'name': 'name', 'type': str, 'doc': 'the name of the constructor argument'}, - is_method=False) - def object_attr(**kwargs): - '''Decorator to override the default mapping scheme for a given object attribute. - - Decorate ObjectMapper methods with this function when extending ObjectMapper to override the default - scheme for mapping between Container and Builder objects. The decorated method should accept as its - first argument the Container object that is being mapped. The method should return the child Builder - object (or scalar if the object attribute corresponds to an AttributeSpec) that represents the - attribute given by *name*. - ''' - name = getargs('name', kwargs) - return _object_attr(name) - - @staticmethod - def __is_attr(attr_val): - return hasattr(attr_val, _obj_attr) - - @staticmethod - def __get_obj_attr(attr_val): - return getattr(attr_val, _obj_attr) - - @staticmethod - def __is_constructor_arg(attr_val): - return hasattr(attr_val, _const_arg) - - @staticmethod - def __get_cargname(attr_val): - return getattr(attr_val, _const_arg) - - @ExtenderMeta.post_init - def __gather_procedures(cls, name, bases, classdict): - if hasattr(cls, 'constructor_args'): - cls.constructor_args = copy(cls.constructor_args) - else: - cls.constructor_args = dict() - if hasattr(cls, 'obj_attrs'): - cls.obj_attrs = copy(cls.obj_attrs) - else: - cls.obj_attrs = dict() - for name, func in cls.__dict__.items(): - if cls.__is_constructor_arg(func): - cls.constructor_args[cls.__get_cargname(func)] = getattr(cls, name) - elif cls.__is_attr(func): - cls.obj_attrs[cls.__get_obj_attr(func)] = getattr(cls, name) - - @docval({'name': 'spec', 'type': (DatasetSpec, GroupSpec), - 'doc': 'The specification for mapping objects to builders'}) - def __init__(self, **kwargs): - """ Create a map from Container attributes to NWB specifications """ - spec = getargs('spec', kwargs) - self.__spec = spec - self.__data_type_key = spec.type_key() - self.__spec2attr = dict() - self.__attr2spec = dict() - self.__spec2carg = dict() - self.__carg2spec = dict() - self.__map_spec(spec) - - @property - def spec(self): - ''' the Spec used in this ObjectMapper ''' - return self.__spec - - @_constructor_arg('name') - def get_container_name(self, *args): - builder = args[0] - return builder.name - - @classmethod - @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to get the name for'}) - def convert_dt_name(cls, **kwargs): - '''Get the attribute name corresponding to a specification''' - spec = getargs('spec', kwargs) - if isinstance(spec, LinkSpec): - name = spec.target_type - else: - if spec.data_type_def is not None: - name = spec.data_type_def - elif spec.data_type_inc is not None: - name = spec.data_type_inc - else: - raise ValueError('found spec without name or data_type') - s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) - name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() - if name[-1] != 's' and spec.is_many(): - name += 's' - return name - - @classmethod - def __get_fields(cls, name_stack, all_names, spec): - name = spec.name - if spec.name is None: - name = cls.convert_dt_name(spec) - name_stack.append(name) - if name in all_names: - name = "_".join(name_stack) - all_names[name] = spec - if isinstance(spec, BaseStorageSpec): - if not (spec.data_type_def is None and spec.data_type_inc is None): - # don't get names for components in data_types - return - for subspec in spec.attributes: - cls.__get_fields(name_stack, all_names, subspec) - if isinstance(spec, GroupSpec): - for subspec in spec.datasets: - cls.__get_fields(name_stack, all_names, subspec) - for subspec in spec.groups: - cls.__get_fields(name_stack, all_names, subspec) - for subspec in spec.links: - cls.__get_fields(name_stack, all_names, subspec) - name_stack.pop() - - @classmethod - @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to get the object attribute names for'}) - def get_attr_names(cls, **kwargs): - '''Get the attribute names for each subspecification in a Spec''' - spec = getargs('spec', kwargs) - names = OrderedDict() - for subspec in spec.attributes: - cls.__get_fields(list(), names, subspec) - if isinstance(spec, GroupSpec): - for subspec in spec.groups: - cls.__get_fields(list(), names, subspec) - for subspec in spec.datasets: - cls.__get_fields(list(), names, subspec) - for subspec in spec.links: - cls.__get_fields(list(), names, subspec) - return names - - def __map_spec(self, spec): - attr_names = self.get_attr_names(spec) - for k, v in attr_names.items(): - self.map_spec(k, v) - - @docval({"name": "attr_name", "type": str, "doc": "the name of the object to map"}, - {"name": "spec", "type": Spec, "doc": "the spec to map the attribute to"}) - def map_attr(self, **kwargs): - """ Map an attribute to spec. Use this to override default behavior """ - attr_name, spec = getargs('attr_name', 'spec', kwargs) - if hasattr(spec, 'name') and spec.name is not None: - n = spec.name - elif hasattr(spec, 'data_type_def') and spec.data_type_def is not None: - n = spec.data_type_def # noqa: F841 - self.__spec2attr[spec] = attr_name - self.__attr2spec[attr_name] = spec - - @docval({"name": "attr_name", "type": str, "doc": "the name of the attribute"}) - def get_attr_spec(self, **kwargs): - """ Return the Spec for a given attribute """ - attr_name = getargs('attr_name', kwargs) - return self.__attr2spec.get(attr_name) - - @docval({"name": "carg_name", "type": str, "doc": "the name of the constructor argument"}) - def get_carg_spec(self, **kwargs): - """ Return the Spec for a given constructor argument """ - carg_name = getargs('carg_name', kwargs) - return self.__attr2spec.get(carg_name) - - @docval({"name": "const_arg", "type": str, "doc": "the name of the constructor argument to map"}, - {"name": "spec", "type": Spec, "doc": "the spec to map the attribute to"}) - def map_const_arg(self, **kwargs): - """ Map an attribute to spec. Use this to override default behavior """ - const_arg, spec = getargs('const_arg', 'spec', kwargs) - self.__spec2carg[spec] = const_arg - self.__carg2spec[const_arg] = spec - - @docval({"name": "spec", "type": Spec, "doc": "the spec to map the attribute to"}) - def unmap(self, **kwargs): - """ Removing any mapping for a specification. Use this to override default mapping """ - spec = getargs('spec', kwargs) - self.__spec2attr.pop(spec, None) - self.__spec2carg.pop(spec, None) - - @docval({"name": "attr_carg", "type": str, "doc": "the constructor argument/object attribute to map this spec to"}, - {"name": "spec", "type": Spec, "doc": "the spec to map the attribute to"}) - def map_spec(self, **kwargs): - """ Map the given specification to the construct argument and object attribute """ - spec, attr_carg = getargs('spec', 'attr_carg', kwargs) - self.map_const_arg(attr_carg, spec) - self.map_attr(attr_carg, spec) - - def __get_override_carg(self, *args): - name = args[0] - remaining_args = tuple(args[1:]) - if name in self.constructor_args: - func = self.constructor_args[name] - try: - # remaining_args is [builder, manager] - return func(self, *remaining_args) - except TypeError: - # LEGACY: remaining_args is [manager] - return func(self, *remaining_args[:-1]) - return None - - def __get_override_attr(self, name, container, manager): - if name in self.obj_attrs: - func = self.obj_attrs[name] - return func(self, container, manager) - return None - - @docval({"name": "spec", "type": Spec, "doc": "the spec to get the attribute for"}, - returns='the attribute name', rtype=str) - def get_attribute(self, **kwargs): - ''' Get the object attribute name for the given Spec ''' - spec = getargs('spec', kwargs) - val = self.__spec2attr.get(spec, None) - return val - - @docval({"name": "spec", "type": Spec, "doc": "the spec to get the attribute value for"}, - {"name": "container", "type": Container, "doc": "the container to get the attribute value from"}, - {"name": "manager", "type": BuildManager, "doc": "the BuildManager used for managing this build"}, - returns='the value of the attribute') - def get_attr_value(self, **kwargs): - ''' Get the value of the attribute corresponding to this spec from the given container ''' - spec, container, manager = getargs('spec', 'container', 'manager', kwargs) - attr_name = self.get_attribute(spec) - if attr_name is None: - return None - attr_val = self.__get_override_attr(attr_name, container, manager) - if attr_val is None: - # TODO: A message like this should be used to warn users when an expected attribute - # does not exist on a Container object - # - # if not hasattr(container, attr_name): - # msg = "Container '%s' (%s) does not have attribute '%s'" \ - # % (container.name, type(container), attr_name) - # #warnings.warn(msg) - attr_val = getattr(container, attr_name, None) - if attr_val is not None: - attr_val = self.__convert_value(attr_val, spec) - return attr_val - - def __convert_value(self, value, spec): - ret = value - if isinstance(spec, AttributeSpec): - if 'text' in spec.dtype: - if spec.shape is not None: - ret = list(map(text_type, value)) - else: - ret = text_type(value) - elif isinstance(spec, DatasetSpec): - # TODO: make sure we can handle specs with data_type_inc set - if spec.data_type_inc is not None: - ret = value - else: - if spec.dtype is not None: - string_type = None - if 'text' in spec.dtype: - string_type = text_type - elif 'ascii' in spec.dtype: - string_type = binary_type - elif 'isodatetime' in spec.dtype: - string_type = datetime.isoformat - if string_type is not None: - if spec.dims is not None: - ret = list(map(string_type, value)) - else: - ret = string_type(value) - return ret - - @docval({"name": "spec", "type": Spec, "doc": "the spec to get the constructor argument for"}, - returns="the name of the constructor argument", rtype=str) - def get_const_arg(self, **kwargs): - ''' Get the constructor argument for the given Spec ''' - spec = getargs('spec', kwargs) - return self.__spec2carg.get(spec, None) - - @docval({"name": "container", "type": Container, "doc": "the container to convert to a Builder"}, - {"name": "manager", "type": BuildManager, "doc": "the BuildManager to use for managing this build"}, - {"name": "parent", "type": Builder, "doc": "the parent of the resulting Builder", 'default': None}, - {"name": "source", "type": str, - "doc": "the source of container being built i.e. file path", 'default': None}, - {"name": "builder", "type": GroupBuilder, "doc": "the Builder to build on", 'default': None}, - {"name": "spec_ext", "type": Spec, "doc": "a spec extension", 'default': None}, - returns="the Builder representing the given Container", rtype=Builder) - def build(self, **kwargs): - ''' Convert a Container to a Builder representation ''' - container, manager, parent, source = getargs('container', 'manager', 'parent', 'source', kwargs) - builder = getargs('builder', kwargs) - name = manager.get_builder_name(container) - if isinstance(self.__spec, GroupSpec): - if builder is None: - builder = GroupBuilder(name, parent=parent, source=source) - self.__add_datasets(builder, self.__spec.datasets, container, manager, source) - self.__add_groups(builder, self.__spec.groups, container, manager, source) - self.__add_links(builder, self.__spec.links, container, manager, source) - else: - if not isinstance(container, Data): - msg = "'container' must be of type Data with DatasetSpec" - raise ValueError(msg) - if isinstance(self.spec.dtype, RefSpec): - bldr_data = self.__get_ref_builder(self.spec.dtype, self.spec.shape, container, manager) - try: - bldr_data, dtype = self.convert_dtype(self.spec, bldr_data) - except Exception as ex: - msg = 'could not resolve dtype for %s \'%s\'' % (type(container).__name__, container.name) - raise_from(Exception(msg), ex) - builder = DatasetBuilder(name, bldr_data, parent=parent, source=source, dtype=dtype) - elif isinstance(self.spec.dtype, list): - refs = [(i, subt) for i, subt in enumerate(self.spec.dtype) if isinstance(subt.dtype, RefSpec)] - bldr_data = copy(container.data) - bldr_data = list() - for i, row in enumerate(container.data): - tmp = list(row) - for j, subt in refs: - tmp[j] = self.__get_ref_builder(subt.dtype, None, row[j], manager) - bldr_data.append(tuple(tmp)) - try: - bldr_data, dtype = self.convert_dtype(self.spec, bldr_data) - except Exception as ex: - msg = 'could not resolve dtype for %s \'%s\'' % (type(container).__name__, container.name) - raise_from(Exception(msg), ex) - builder = DatasetBuilder(name, bldr_data, parent=parent, source=source, dtype=dtype) - else: - if self.__spec.dtype is None and self.__is_reftype(container.data): - bldr_data = list() - for d in container.data: - bldr_data.append(ReferenceBuilder(manager.build(d))) - builder = DatasetBuilder(name, bldr_data, parent=parent, source=source, - dtype='object') - else: - try: - bldr_data, dtype = self.convert_dtype(self.spec, container.data) - except Exception as ex: - msg = 'could not resolve dtype for %s \'%s\'' % (type(container).__name__, container.name) - raise_from(Exception(msg), ex) - builder = DatasetBuilder(name, bldr_data, parent=parent, source=source, dtype=dtype) - self.__add_attributes(builder, self.__spec.attributes, container, manager, source) - return builder - - def __is_reftype(self, data): - tmp = data - while hasattr(tmp, '__len__') and not isinstance(tmp, (Container, text_type, binary_type)): - tmptmp = None - for t in tmp: - # In case of a numeric array stop the iteration at the first element to avoid long-running loop - if isinstance(t, (integer_types, float, complex, bool)): - break - if hasattr(t, '__len__') and not isinstance(t, (Container, text_type, binary_type)) and len(t) > 0: - tmptmp = tmp[0] - break - if tmptmp is not None: - break - else: - tmp = tmp[0] - if isinstance(tmp, Container): - return True - else: - return False - - def __get_ref_builder(self, dtype, shape, container, manager): - bldr_data = None - if dtype.is_region(): - if shape is None: - if not isinstance(container, DataRegion): - msg = "'container' must be of type DataRegion if spec represents region reference" - raise ValueError(msg) - bldr_data = RegionBuilder(container.region, manager.build(container.data)) - else: - bldr_data = list() - for d in container.data: - bldr_data.append(RegionBuilder(d.slice, manager.build(d.target))) - else: - if shape is None: - if isinstance(container, Container): - bldr_data = ReferenceBuilder(manager.build(container)) - else: - bldr_data = ReferenceBuilder(manager.build(container.data)) - else: - bldr_data = list() - for d in container.data: - bldr_data.append(ReferenceBuilder(manager.build(d.target))) - return bldr_data - - def __is_null(self, item): - if item is None: - return True - else: - if any(isinstance(item, t) for t in (list, tuple, dict, set)): - return len(item) == 0 - return False - - def __add_attributes(self, builder, attributes, container, build_manager, source): - for spec in attributes: - if spec.value is not None: - attr_value = spec.value - else: - attr_value = self.get_attr_value(spec, container, build_manager) - if attr_value is None: - attr_value = spec.default_value - - if isinstance(spec.dtype, RefSpec): - if not self.__is_reftype(attr_value): - if attr_value is None: - msg = "object of data_type %s not found on %s '%s'" % \ - (spec.dtype.target_type, type(container).__name__, container.name) - else: - msg = "invalid type for reference '%s' (%s) - must be Container" % (spec.name, type(attr_value)) - raise ValueError(msg) - target_builder = build_manager.build(attr_value, source=source) - attr_value = ReferenceBuilder(target_builder) - else: - if attr_value is not None: - try: - attr_value, attr_dtype = self.convert_dtype(spec, attr_value) - except Exception as ex: - msg = 'could not convert %s for %s %s' % (spec.name, type(container).__name__, container.name) - raise_from(Exception(msg), ex) - - # do not write empty or null valued objects - if attr_value is None: - if spec.required: - msg = "attribute '%s' for '%s' (%s)"\ - % (spec.name, builder.name, self.spec.data_type_def) - warnings.warn(msg, MissingRequiredWarning) - continue - - builder.set_attribute(spec.name, attr_value) - - def __add_links(self, builder, links, container, build_manager, source): - for spec in links: - attr_value = self.get_attr_value(spec, container, build_manager) - if not attr_value: - continue - self.__add_containers(builder, spec, attr_value, build_manager, source, container) - - def __is_empty(self, val): - if val is None: - return True - if isinstance(val, DataIO): - val = val.data - if isinstance(val, AbstractDataChunkIterator): - return False - else: - if (hasattr(val, '__len__') and len(val) == 0): - return True - else: - return False - - def __add_datasets(self, builder, datasets, container, build_manager, source): - for spec in datasets: - attr_value = self.get_attr_value(spec, container, build_manager) - # TODO: add check for required datasets - if self.__is_empty(attr_value): - if spec.required: - msg = "dataset '%s' for '%s' of type (%s)"\ - % (spec.name, builder.name, self.spec.data_type_def) - warnings.warn(msg, MissingRequiredWarning) - continue - if isinstance(attr_value, Builder): - builder.set_builder(attr_value) - elif spec.data_type_def is None and spec.data_type_inc is None: - if spec.name in builder.datasets: - sub_builder = builder.datasets[spec.name] - else: - try: - data, dtype = self.convert_dtype(spec, attr_value) - except Exception as ex: - msg = 'could not convert \'%s\' for %s \'%s\'' - msg = msg % (spec.name, type(container).__name__, container.name) - raise_from(Exception(msg), ex) - sub_builder = builder.add_dataset(spec.name, data, dtype=dtype) - self.__add_attributes(sub_builder, spec.attributes, container, build_manager, source) - else: - self.__add_containers(builder, spec, attr_value, build_manager, source, container) - - def __add_groups(self, builder, groups, container, build_manager, source): - for spec in groups: - if spec.data_type_def is None and spec.data_type_inc is None: - # we don't need to get attr_name since any named - # group does not have the concept of value - sub_builder = builder.groups.get(spec.name) - if sub_builder is None: - sub_builder = GroupBuilder(spec.name, source=source) - self.__add_attributes(sub_builder, spec.attributes, container, build_manager, source) - self.__add_datasets(sub_builder, spec.datasets, container, build_manager, source) - - # handle subgroups that are not Containers - attr_name = self.get_attribute(spec) - if attr_name is not None: - attr_value = getattr(container, attr_name, None) - attr_value = self.get_attr_value(spec, container, build_manager) - if any(isinstance(attr_value, t) for t in (list, tuple, set, dict)): - it = iter(attr_value) - if isinstance(attr_value, dict): - it = iter(attr_value.values()) - for item in it: - if isinstance(item, Container): - self.__add_containers(sub_builder, spec, item, build_manager, source, container) - self.__add_groups(sub_builder, spec.groups, container, build_manager, source) - empty = sub_builder.is_empty() - if not empty or (empty and isinstance(spec.quantity, int)): - if sub_builder.name not in builder.groups: - builder.set_group(sub_builder) - else: - if spec.data_type_def is not None: - attr_name = self.get_attribute(spec) - if attr_name is not None: - attr_value = getattr(container, attr_name, None) - if attr_value is not None: - self.__add_containers(builder, spec, attr_value, build_manager, source, container) - else: - attr_name = self.get_attribute(spec) - attr_value = getattr(container, attr_name, None) - if attr_value is not None: - self.__add_containers(builder, spec, attr_value, build_manager, source, container) - - def __add_containers(self, builder, spec, value, build_manager, source, parent_container): - if isinstance(value, Container): - if value.parent is None: - msg = "'%s' (%s) for '%s' (%s)"\ - % (value.name, getattr(value, self.spec.type_key()), - builder.name, self.spec.data_type_def) - warnings.warn(msg, OrphanContainerWarning) - if value.modified: # writing a new container - rendered_obj = build_manager.build(value, source=source) - # use spec to determine what kind of HDF5 - # object this Container corresponds to - if isinstance(spec, LinkSpec) or value.parent is not parent_container: - name = spec.name - builder.set_link(LinkBuilder(rendered_obj, name, builder)) - elif isinstance(spec, DatasetSpec): - if rendered_obj.dtype is None and spec.dtype is not None: - val, dtype = self.convert_dtype(spec, None) - rendered_obj.dtype = dtype - builder.set_dataset(rendered_obj) - else: - builder.set_group(rendered_obj) - elif value.container_source: # make a link to an existing container - if value.container_source != parent_container.container_source or\ - value.parent is not parent_container: - rendered_obj = build_manager.build(value, source=source) - builder.set_link(LinkBuilder(rendered_obj, name=spec.name, parent=builder)) - else: - raise ValueError("Found unmodified Container with no source - '%s' with parent '%s'" % - (value.name, parent_container.name)) - else: - if any(isinstance(value, t) for t in (list, tuple)): - values = value - elif isinstance(value, dict): - values = value.values() - else: - msg = ("received %s, expected Container - 'value' " - "must be an Container a list/tuple/dict of " - "Containers if 'spec' is a GroupSpec") - raise ValueError(msg % value.__class__.__name__) - for container in values: - if container: - self.__add_containers(builder, spec, container, build_manager, source, parent_container) - - def __get_subspec_values(self, builder, spec, manager): - ret = dict() - # First get attributes - attributes = builder.attributes - for attr_spec in spec.attributes: - attr_val = attributes.get(attr_spec.name) - if attr_val is None: - continue - if isinstance(attr_val, (GroupBuilder, DatasetBuilder)): - ret[attr_spec] = manager.construct(attr_val) - elif isinstance(attr_val, RegionBuilder): - raise ValueError("RegionReferences as attributes is not yet supported") - elif isinstance(attr_val, ReferenceBuilder): - ret[attr_spec] = manager.construct(attr_val.builder) - else: - ret[attr_spec] = attr_val - if isinstance(spec, GroupSpec): - if not isinstance(builder, GroupBuilder): - raise ValueError("__get_subspec_values - must pass GroupBuilder with GroupSpec") - # first aggregate links by data type and separate them - # by group and dataset - groups = dict(builder.groups) # make a copy so we can separate links - datasets = dict(builder.datasets) # make a copy so we can separate links - links = builder.links - link_dt = dict() - for link_builder in links.values(): - target = link_builder.builder - if isinstance(target, DatasetBuilder): - datasets[link_builder.name] = target - else: - groups[link_builder.name] = target - dt = manager.get_builder_dt(target) - if dt is not None: - link_dt.setdefault(dt, list()).append(target) - # now assign links to their respective specification - for subspec in spec.links: - if subspec.name is not None: - sub_builder = links.get(subspec.name) - if sub_builder is not None: - ret[subspec] = manager.construct(sub_builder.builder) - else: - sub_builder = link_dt.get(subspec.target_type) - if sub_builder is not None: - ret[subspec] = self.__flatten(sub_builder, subspec, manager) - # now process groups and datasets - self.__get_sub_builders(groups, spec.groups, manager, ret) - self.__get_sub_builders(datasets, spec.datasets, manager, ret) - elif isinstance(spec, DatasetSpec): - if not isinstance(builder, DatasetBuilder): - raise ValueError("__get_subspec_values - must pass DatasetBuilder with DatasetSpec") - ret[spec] = builder.data - return ret - - def __get_sub_builders(self, sub_builders, subspecs, manager, ret): - # index builders by data_type - builder_dt = dict() - for g in sub_builders.values(): - dt = manager.get_builder_dt(g) - ns = manager.get_builder_ns(g) - if dt is None or ns is None: - continue - for parent_dt in manager.namespace_catalog.get_hierarchy(ns, dt): - builder_dt.setdefault(parent_dt, list()).append(g) - for subspec in subspecs: - # first get data type for the spec - if subspec.data_type_def is not None: - dt = subspec.data_type_def - elif subspec.data_type_inc is not None: - dt = subspec.data_type_inc - else: - dt = None - # use name if we can, otherwise use data_data - if subspec.name is None: - sub_builder = builder_dt.get(dt) - if sub_builder is not None: - sub_builder = self.__flatten(sub_builder, subspec, manager) - ret[subspec] = sub_builder - else: - sub_builder = sub_builders.get(subspec.name) - if sub_builder is None: - continue - if dt is None: - # recurse - ret.update(self.__get_subspec_values(sub_builder, subspec, manager)) - else: - ret[subspec] = manager.construct(sub_builder) - - def __flatten(self, sub_builder, subspec, manager): - """ - Convert one-or-many to a single object or a list, - depending on the spec - """ - tmp = [manager.construct(b) for b in sub_builder] - if len(tmp) == 1 and not subspec.is_many(): - tmp = tmp[0] - return tmp - - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), - 'doc': 'the builder to construct the Container from'}, - {'name': 'manager', 'type': BuildManager, 'doc': 'the BuildManager for this build'}) - def construct(self, **kwargs): - ''' Construct an Container from the given Builder ''' - builder, manager = getargs('builder', 'manager', kwargs) - cls = manager.get_cls(builder) - # gather all subspecs - subspecs = self.__get_subspec_values(builder, self.spec, manager) - # get the constructor argument that each specification corresponds to - const_args = dict() - for subspec, value in subspecs.items(): - const_arg = self.get_const_arg(subspec) - if const_arg is not None: - if isinstance(subspec, BaseStorageSpec) and subspec.is_many(): - existing_value = const_args.get(const_arg) - if isinstance(existing_value, list): - value = existing_value + value - const_args[const_arg] = value - # build kwargs for the constructor - kwargs = dict() - for const_arg in get_docval(cls.__init__): - argname = const_arg['name'] - override = self.__get_override_carg(argname, builder, manager) - if override is not None: - val = override - elif argname in const_args: - val = const_args[argname] - else: - continue - kwargs[argname] = val - try: - obj = cls(**kwargs) - obj.container_source = builder.source - except Exception as ex: - msg = 'Could not construct %s object' % (cls.__name__,) - raise_from(Exception(msg), ex) - return obj - - @docval({'name': 'container', 'type': Container, 'doc': 'the Container to get the Builder name for'}) - def get_builder_name(self, **kwargs): - '''Get the name of a Builder that represents a Container''' - container = getargs('container', kwargs) - if self.__spec.name not in (NAME_WILDCARD, None): - ret = self.__spec.name - else: - if container.name is None: - if self.__spec.default_name is not None: - ret = self.__spec.default_name - else: - msg = 'Unable to determine name of container type %s' % self.__spec.data_type_def - raise ValueError(msg) - else: - ret = container.name - return ret - - -class TypeSource(object): - '''A class to indicate the source of a data_type in a namespace. - - This class should only be used by TypeMap - ''' - - @docval({"name": "namespace", "type": str, "doc": "the namespace the from, which the data_type originated"}, - {"name": "data_type", "type": str, "doc": "the name of the type"}) - def __init__(self, **kwargs): - namespace, data_type = getargs('namespace', 'data_type', kwargs) - self.__namespace = namespace - self.__data_type = data_type - - @property - def namespace(self): - return self.__namespace - - @property - def data_type(self): - return self.__data_type - - -class TypeMap(object): - ''' A class to maintain the map between ObjectMappers and Container classes - ''' - - @docval({'name': 'namespaces', 'type': NamespaceCatalog, 'doc': 'the NamespaceCatalog to use'}, - {'name': 'mapper_cls', 'type': type, 'doc': 'the ObjectMapper class to use', 'default': ObjectMapper}) - def __init__(self, **kwargs): - namespaces = getargs('namespaces', kwargs) - self.__ns_catalog = namespaces - self.__mappers = dict() # already constructed ObjectMapper classes - self.__mapper_cls = dict() # the ObjectMapper class to use for each container type - self.__container_types = OrderedDict() - self.__data_types = dict() - self.__default_mapper_cls = getargs('mapper_cls', kwargs) - - @property - def namespace_catalog(self): - return self.__ns_catalog - - def __copy__(self): - ret = TypeMap(copy(self.__ns_catalog), self.__default_mapper_cls) - ret.merge(self) - return ret - - def __deepcopy__(self, memo): - # XXX: From @nicain: All of a sudden legacy tests started - # needing this argument in deepcopy. Doesn't hurt anything, though. - return self.__copy__() - - def copy_mappers(self, type_map): - for namespace in self.__ns_catalog.namespaces: - if namespace not in type_map.__container_types: - continue - for data_type in self.__ns_catalog.get_namespace(namespace).get_registered_types(): - container_cls = type_map.__container_types[namespace].get(data_type) - if container_cls is None: - continue - self.register_container_type(namespace, data_type, container_cls) - if container_cls in type_map.__mapper_cls: - self.register_map(container_cls, type_map.__mapper_cls[container_cls]) - - def merge(self, type_map): - for namespace in type_map.__container_types: - for data_type in type_map.__container_types[namespace]: - - container_cls = type_map.__container_types[namespace][data_type] - self.register_container_type(namespace, data_type, container_cls) - - for container_cls in type_map.__mapper_cls: - self.register_map(container_cls, type_map.__mapper_cls[container_cls]) - - @docval({'name': 'namespace_path', 'type': str, 'doc': 'the path to the file containing the namespaces(s) to load'}, - {'name': 'resolve', 'type': bool, - 'doc': 'whether or not to include objects from included/parent spec objects', 'default': True}, - {'name': 'reader', - 'type': SpecReader, - 'doc': 'the class to user for reading specifications', 'default': None}, - returns="the namespaces loaded from the given file", rtype=tuple) - def load_namespaces(self, **kwargs): - '''Load namespaces from a namespace file. - - This method will call load_namespaces on the NamespaceCatalog used to construct this TypeMap. Additionally, - it will process the return value to keep track of what types were included in the loaded namespaces. Calling - load_namespaces here has the advantage of being able to keep track of type dependencies across namespaces. - ''' - deps = call_docval_func(self.__ns_catalog.load_namespaces, kwargs) - for new_ns, ns_deps in deps.items(): - for src_ns, types in ns_deps.items(): - for dt in types: - container_cls = self.get_container_cls(src_ns, dt) - if container_cls is None: - container_cls = TypeSource(src_ns, dt) - self.register_container_type(new_ns, dt, container_cls) - return tuple(deps.keys()) - - _type_map = { - 'text': str, - 'float': float, - 'float64': float, - 'int': int, - 'int32': int, - 'isodatetime': datetime - } - - def __get_type(self, spec): - if isinstance(spec, AttributeSpec): - if isinstance(spec.dtype, RefSpec): - tgttype = spec.dtype.target_type - for val in self.__container_types.values(): - container_type = val.get(tgttype) - if container_type is not None: - return container_type - return (Data, Container) - elif spec.shape is None: - return self._type_map.get(spec.dtype) - else: - return ('array_data',) - elif isinstance(spec, LinkSpec): - return Container - else: - if not (spec.data_type_inc is None and spec.data_type_inc is None): - if spec.name is not None: - return (list, tuple, dict, set) - else: - return Container - else: - return ('array_data', 'data',) - - def __get_constructor(self, base, addl_fields): - # TODO: fix this to be more maintainable and smarter - existing_args = set() - docval_args = list() - new_args = list() - if base is not None: - for arg in get_docval(base.__init__): - existing_args.add(arg['name']) - if arg['name'] in addl_fields: - continue - docval_args.append(arg) - for f, field_spec in addl_fields.items(): - dtype = self.__get_type(field_spec) - docval_arg = {'name': f, 'type': dtype, 'doc': field_spec.doc} - if not field_spec.required: - docval_arg['default'] = getattr(field_spec, 'default_value', None) - docval_args.append(docval_arg) - if f not in existing_args: - new_args.append(f) - # TODO: set __nwbfields__ - if base is None: - @docval(*docval_args) - def __init__(self, **kwargs): - for f in new_args: - setattr(self, f, kwargs.get(f, None)) - return __init__ - else: - @docval(*docval_args) - def __init__(self, **kwargs): - pargs, pkwargs = fmt_docval_args(base.__init__, kwargs) - super(type(self), self).__init__(*pargs, **pkwargs) - for f in new_args: - setattr(self, f, kwargs.get(f, None)) - return __init__ - - @docval({"name": "namespace", "type": str, "doc": "the namespace containing the data_type"}, - {"name": "data_type", "type": str, "doc": "the data type to create a Container class for"}, - returns='the class for the given namespace and data_type', rtype=type) - def get_container_cls(self, **kwargs): - '''Get the container class from data type specification - - If no class has been associated with the ``data_type`` from ``namespace``, - a class will be dynamically created and returned. - ''' - namespace, data_type = getargs('namespace', 'data_type', kwargs) - cls = self.__get_container_cls(namespace, data_type) - if cls is None: - spec = self.__ns_catalog.get_spec(namespace, data_type) - dt_hier = self.__ns_catalog.get_hierarchy(namespace, data_type) - parent_cls = None - for t in dt_hier: - parent_cls = self.__get_container_cls(namespace, t) - if parent_cls is not None: - break - bases = tuple() - if parent_cls is not None: - bases = (parent_cls,) - else: - if isinstance(spec, GroupSpec): - bases = (Container,) - elif isinstance(spec, DatasetSpec): - bases = (Data,) - else: - raise ValueError("Cannot generate class from %s" % type(spec)) - parent_cls = bases[0] - name = data_type - attr_names = self.__default_mapper_cls.get_attr_names(spec) - fields = dict() - for k, field_spec in attr_names.items(): - if not spec.is_inherited_spec(field_spec): - fields[k] = field_spec - d = {'__init__': self.__get_constructor(parent_cls, fields)} - cls = type(str(name), bases, d) - self.register_container_type(namespace, data_type, cls) - return cls - - def __get_container_cls(self, namespace, data_type): - if namespace not in self.__container_types: - return None - if data_type not in self.__container_types[namespace]: - return None - ret = self.__container_types[namespace][data_type] - if isinstance(ret, TypeSource): - ret = self.__get_container_cls(ret.namespace, ret.data_type) - if ret is not None: - self.register_container_type(namespace, data_type, ret) - return ret - - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), - 'doc': 'the builder to get the data_type for'}) - def get_builder_dt(self, **kwargs): - ''' - Get the data_type of a builder - ''' - builder = getargs('builder', kwargs) - ret = builder.attributes.get(self.__ns_catalog.group_spec_cls.type_key()) - if isinstance(ret, bytes): - ret = ret.decode('UTF-8') - return ret - - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), - 'doc': 'the builder to get the sub-specification for'}) - def get_builder_ns(self, **kwargs): - ''' - Get the namespace of a builder - ''' - builder = getargs('builder', kwargs) - if isinstance(builder, LinkBuilder): - builder = builder.builder - ret = builder.attributes.get('namespace') - return ret - - @docval({'name': 'builder', 'type': Builder, - 'doc': 'the Builder object to get the corresponding Container class for'}) - def get_cls(self, **kwargs): - ''' Get the class object for the given Builder ''' - builder = getargs('builder', kwargs) - data_type = self.get_builder_dt(builder) - if data_type is None: - raise ValueError("No data_type found for builder %s" % builder.path) - namespace = self.get_builder_ns(builder) - if namespace is None: - raise ValueError("No namespace found for builder %s" % builder.path) - return self.get_container_cls(namespace, data_type) - - @docval({'name': 'spec', 'type': (DatasetSpec, GroupSpec), 'doc': 'the parent spec to search'}, - {'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), - 'doc': 'the builder to get the sub-specification for'}) - def get_subspec(self, **kwargs): - ''' - Get the specification from this spec that corresponds to the given builder - ''' - spec, builder = getargs('spec', 'builder', kwargs) - if isinstance(builder, LinkBuilder): - builder_type = type(builder.builder) - else: - builder_type = type(builder) - if issubclass(builder_type, DatasetBuilder): - subspec = spec.get_dataset(builder.name) - else: - subspec = spec.get_group(builder.name) - if subspec is None: - # builder was generated from something with a data_type and a wildcard name - if isinstance(builder, LinkBuilder): - dt = self.get_builder_dt(builder.builder) - else: - dt = self.get_builder_dt(builder) - if dt is not None: - ns = self.get_builder_ns(builder) - hierarchy = self.__ns_catalog.get_hierarchy(ns, dt) - for t in hierarchy: - subspec = spec.get_data_type(t) - if subspec is not None: - break - return subspec - - def get_container_ns_dt(self, obj): - container_cls = obj.__class__ - namespace, data_type = self.get_container_cls_dt(container_cls) - return namespace, data_type - - def get_container_cls_dt(self, cls): - return self.__data_types.get(cls, (None, None)) - - @docval({'name': 'namespace', 'type': str, - 'doc': 'the namespace to get the container classes for', 'default': None}) - def get_container_classes(self, **kwargs): - namespace = getargs('namespace', kwargs) - ret = self.__data_types.keys() - if namespace is not None: - ret = filter(lambda x: self.__data_types[x][0] == namespace, ret) - return list(ret) - - @docval({'name': 'obj', 'type': (Container, Builder), 'doc': 'the object to get the ObjectMapper for'}, - returns='the ObjectMapper to use for mapping the given object', rtype='ObjectMapper') - def get_map(self, **kwargs): - """ Return the ObjectMapper object that should be used for the given container """ - obj = getargs('obj', kwargs) - # get the container class, and namespace/data_type - if isinstance(obj, Container): - container_cls = obj.__class__ - namespace, data_type = self.get_container_ns_dt(obj) - if namespace is None: - raise ValueError("class %s is not mapped to a data_type" % container_cls) - else: - data_type = self.get_builder_dt(obj) - namespace = self.get_builder_ns(obj) - container_cls = self.get_cls(obj) - # now build the ObjectMapper class - spec = self.__ns_catalog.get_spec(namespace, data_type) - mapper = self.__mappers.get(container_cls) - if mapper is None: - mapper_cls = self.__default_mapper_cls - for cls in container_cls.__mro__: - tmp_mapper_cls = self.__mapper_cls.get(cls) - if tmp_mapper_cls is not None: - mapper_cls = tmp_mapper_cls - break - - mapper = mapper_cls(spec) - self.__mappers[container_cls] = mapper - return mapper - - @docval({"name": "namespace", "type": str, "doc": "the namespace containing the data_type to map the class to"}, - {"name": "data_type", "type": str, "doc": "the data_type to map the class to"}, - {"name": "container_cls", "type": (TypeSource, type), "doc": "the class to map to the specified data_type"}) - def register_container_type(self, **kwargs): - ''' Map a container class to a data_type ''' - namespace, data_type, container_cls = getargs('namespace', 'data_type', 'container_cls', kwargs) - spec = self.__ns_catalog.get_spec(namespace, data_type) # make sure the spec exists - self.__container_types.setdefault(namespace, dict()) - self.__container_types[namespace][data_type] = container_cls - self.__data_types.setdefault(container_cls, (namespace, data_type)) - setattr(container_cls, spec.type_key(), data_type) - setattr(container_cls, 'namespace', namespace) - - @docval({"name": "container_cls", "type": type, - "doc": "the Container class for which the given ObjectMapper class gets used for"}, - {"name": "mapper_cls", "type": type, "doc": "the ObjectMapper class to use to map"}) - def register_map(self, **kwargs): - ''' Map a container class to an ObjectMapper class ''' - container_cls, mapper_cls = getargs('container_cls', 'mapper_cls', kwargs) - if self.get_container_cls_dt(container_cls) == (None, None): - raise ValueError('cannot register map for type %s - no data_type found' % container_cls) - self.__mapper_cls[container_cls] = mapper_cls - - @docval({"name": "container", "type": Container, "doc": "the container to convert to a Builder"}, - {"name": "manager", "type": BuildManager, - "doc": "the BuildManager to use for managing this build", 'default': None}, - {"name": "source", "type": str, - "doc": "the source of container being built i.e. file path", 'default': None}, - {"name": "builder", "type": GroupBuilder, "doc": "the Builder to build on", 'default': None}) - def build(self, **kwargs): - """ Build the GroupBuilder for the given Container""" - container, manager, builder = getargs('container', 'manager', 'builder', kwargs) - if manager is None: - manager = BuildManager(self) - attr_map = self.get_map(container) - if attr_map is None: - raise ValueError('No ObjectMapper found for container of type %s' % str(container.__class__.__name__)) - else: - builder = attr_map.build(container, manager, builder=builder, source=getargs('source', kwargs)) - namespace, data_type = self.get_container_ns_dt(container) - builder.set_attribute('namespace', namespace) - builder.set_attribute(attr_map.spec.type_key(), data_type) - return builder - - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), - 'doc': 'the builder to construct the Container from'}, - {'name': 'build_manager', 'type': BuildManager, - 'doc': 'the BuildManager for constructing', 'default': None}) - def construct(self, **kwargs): - """ Construct the Container represented by the given builder """ - builder, build_manager = getargs('builder', 'build_manager', kwargs) - if build_manager is None: - build_manager = BuildManager(self) - attr_map = self.get_map(builder) - if attr_map is None: - raise ValueError('No ObjectMapper found for builder of type %s' - % str(container.__class__.__name__)) # noqa: F821 - else: - return attr_map.construct(builder, build_manager) - - @docval({"name": "container", "type": Container, "doc": "the container to convert to a Builder"}, - returns='The name a Builder should be given when building this container', rtype=str) - def get_builder_name(self, **kwargs): - ''' Get the name a Builder should be given ''' - container = getargs('container', kwargs) - attr_map = self.get_map(container) - if attr_map is None: - raise ValueError('No ObjectMapper found for container of type %s' % str(container.__class__.__name__)) - else: - return attr_map.get_builder_name(container) diff --git a/src/pynwb/form/build/warnings.py b/src/pynwb/form/build/warnings.py deleted file mode 100644 index 1dc580481..000000000 --- a/src/pynwb/form/build/warnings.py +++ /dev/null @@ -1,15 +0,0 @@ -class OrphanContainerWarning(UserWarning): - """ - Raised when a container does not have a parent. - - Only the top level container (e.g. NWBFile) should be - without a parent - """ - pass - - -class MissingRequiredWarning(UserWarning): - """ - Raised when a required field is missing. - """ - pass diff --git a/src/pynwb/form/container.py b/src/pynwb/form/container.py deleted file mode 100644 index 6efa579cf..000000000 --- a/src/pynwb/form/container.py +++ /dev/null @@ -1,123 +0,0 @@ -import abc -from six import with_metaclass -from .utils import docval, getargs - - -class Container(with_metaclass(abc.ABCMeta, object)): - - @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'}, - {'name': 'parent', 'type': 'Container', 'doc': 'the Container that holds this Container', 'default': None}, - {'name': 'container_source', 'type': str, 'doc': 'the source of this container', 'default': None}) - def __init__(self, **kwargs): - name = getargs('name', kwargs) - if '/' in name: - raise ValueError("name '" + name + "' cannot contain '/'") - self.__name = name - self.__parent = getargs('parent', kwargs) - self.__container_source = getargs('container_source', kwargs) - self.__children = list() - self.__modified = True - - def __repr__(self): - return "<%s '%s' at 0x%d>" % (self.__class__.__name__, self.name, id(self)) - - @property - def modified(self): - return self.__modified - - @docval({'name': 'modified', 'type': bool, - 'doc': 'whether or not this Container has been modified', 'default': True}) - def set_modified(self, **kwargs): - modified = getargs('modified', kwargs) - self.__modified = modified - if modified and self.parent is not None: - self.parent.set_modified() - - @property - def children(self): - return tuple(self.__children) - - @docval({'name': 'child', 'type': 'Container', - 'doc': 'the child Container for this Container', 'default': None}) - def add_child(self, **kwargs): - child = getargs('child', kwargs) - self.__children.append(child) - self.set_modified() - if not isinstance(child.parent, Container): - child.parent = self - - @classmethod - def type_hierarchy(cls): - return cls.__mro__ - - @property - def name(self): - ''' - The name of this Container - ''' - return self.__name - - @property - def container_source(self): - ''' - The source of this Container - ''' - return self.__container_source - - @container_source.setter - def container_source(self, source): - if self.__container_source is not None: - raise Exception('cannot reassign container_source') - self.__container_source = source - - @property - def parent(self): - ''' - The parent Container of this Container - ''' - return self.__parent - - @parent.setter - def parent(self, parent_container): - if self.__parent is not None: - if isinstance(self.__parent, Container): - raise Exception('cannot reassign parent') - else: - if parent_container is None: - raise ValueError("got None for parent of '%s' - cannot overwrite Proxy with NoneType" % self.name) - if self.__parent.matches(parent_container): - self.__parent = parent_container - else: - self.__parent.add_candidate(parent_container) - else: - self.__parent = parent_container - - -class Data(Container): - - @abc.abstractproperty - def data(self): - ''' - The data that is held by this Container - ''' - pass - - def __nonzero__(self): - return len(self.data) != 0 - - -class DataRegion(Data): - - @abc.abstractproperty - def data(self): - ''' - The target data that this region applies to - ''' - pass - - @abc.abstractproperty - def region(self): - ''' - The region that indexes into data e.g. slice or list of indices - ''' - pass diff --git a/src/pynwb/form/data_utils.py b/src/pynwb/form/data_utils.py deleted file mode 100644 index 6d9079fed..000000000 --- a/src/pynwb/form/data_utils.py +++ /dev/null @@ -1,559 +0,0 @@ -from abc import ABCMeta, abstractmethod, abstractproperty -from collections import Iterable -from operator import itemgetter - -import numpy as np -from six import with_metaclass, text_type, binary_type - -from .container import Data, DataRegion -from .utils import docval, getargs, popargs, docval_macro, get_data_shape - - -def __get_shape_helper(data): - shape = list() - if hasattr(data, '__len__'): - shape.append(len(data)) - if len(data) and not isinstance(data[0], (text_type, binary_type)): - shape.extend(__get_shape_helper(data[0])) - return tuple(shape) - - -def get_shape(data): - if isinstance(data, dict): - return None - elif hasattr(data, '__len__') and not isinstance(data, (text_type, binary_type)): - return __get_shape_helper(data) - else: - return None - - -@docval_macro('array_data') -class AbstractDataChunkIterator(with_metaclass(ABCMeta, object)): - """ - Abstract iterator class used to iterate over DataChunks. - - Derived classes must ensure that all abstract methods and abstract properties are implemented, in - particular, dtype, maxshape, __iter__, ___next__, recommended_chunk_shape, and recommended_data_shape. - """ - - @abstractmethod - def __iter__(self): - """Return the iterator object""" - raise NotImplementedError("__iter__ not implemented for derived class") - - @abstractmethod - def __next__(self): - """ - Return the next data chunk or raise a StopIteration exception if all chunks have been retrieved. - - HINT: numpy.s_ provides a convenient way to generate index tuples using standard array slicing. This - is often useful to define the DataChunk.selection of the current chunk - - :returns: DataChunk object with the data and selection of the current chunk - :rtype: DataChunk - """ - raise NotImplementedError("__next__ not implemented for derived class") - - @abstractmethod - def recommended_chunk_shape(self): - """ - - :return: NumPy-style shape tuple describing the recommended shape for the chunks of the target - array or None. This may or may not be the same as the shape of the chunks returned in the - iteration process. - """ - raise NotImplementedError("recommended_chunk_shape not implemented for derived class") - - @abstractmethod - def recommended_data_shape(self): - """ - Recommend the initial shape for the data array. - - This is useful in particular to avoid repeated resized of the target array when reading from - this data iterator. This should typically be either the final size of the array or the known - minimal shape of the array. - - :return: NumPy-style shape tuple indicating the recommended initial shape for the target array. - This may or may not be the final full shape of the array, i.e., the array is allowed - to grow. This should not be None. - """ - raise NotImplementedError("recommended_data_shape not implemented for derived class") - - @abstractproperty - def dtype(self): - """ - Define the data type of the array - - :return: NumPy style dtype or otherwise compliant dtype string - """ - raise NotImplementedError("dtype not implemented for derived class") - - @abstractproperty - def maxshape(self): - """ - Property describing the maximum shape of the data array that is being iterated over - - :return: NumPy-style shape tuple indicating the maxiumum dimensions up to which the dataset may be - resized. Axes with None are unlimited. - """ - raise NotImplementedError("maxshape not implemented for derived class") - - -class DataChunkIterator(AbstractDataChunkIterator): - """ - Custom iterator class used to iterate over chunks of data. - - This default implementation of AbstractDataChunkIterator accepts any iterable and assumes that we iterate over - the first dimension of the data array. DataChunkIterator supports buffered read, - i.e., multiple values from the input iterator can be combined to a single chunk. This is - useful for buffered I/O operations, e.g., to improve performance by accumulating data - in memory and writing larger blocks at once. - """ - @docval({'name': 'data', 'type': None, 'doc': 'The data object used for iteration', 'default': None}, - {'name': 'maxshape', 'type': tuple, - 'doc': 'The maximum shape of the full data array. Use None to indicate unlimited dimensions', - 'default': None}, - {'name': 'dtype', 'type': np.dtype, 'doc': 'The Numpy data type for the array', 'default': None}, - {'name': 'buffer_size', 'type': int, 'doc': 'Number of values to be buffered in a chunk', 'default': 1}, - ) - def __init__(self, **kwargs): - """Initialize the DataChunkIterator""" - # Get the user parameters - self.data, self.__maxshape, self.__dtype, self.buffer_size = getargs('data', - 'maxshape', - 'dtype', - 'buffer_size', - kwargs) - # Create an iterator for the data if possible - self.__data_iter = iter(self.data) if isinstance(self.data, Iterable) else None - self.__next_chunk = DataChunk(None, None) - self.__first_chunk_shape = None - # Determine the shape of the data if possible - if self.__maxshape is None: - # If the self.data object identifies it shape then use it - if hasattr(self.data, "shape"): - self.__maxshape = self.data.shape - # Avoid the special case of scalar values by making them into a 1D numpy array - if len(self.__maxshape) == 0: - self.data = np.asarray([self.data, ]) - self.__maxshape = self.data.shape - self.__data_iter = iter(self.data) - # Try to get an accurate idea of __maxshape for other Python datastructures if possible. - # Don't just callget_shape for a generator as that would potentially trigger loading of all the data - elif isinstance(self.data, list) or isinstance(self.data, tuple): - self.__maxshape = get_data_shape(self.data, strict_no_data_load=True) - - # If we have a data iterator, then read the first chunk - if self.__data_iter is not None: # and(self.__maxshape is None or self.__dtype is None): - self._read_next_chunk() - - # If we still don't know the shape then try to determine the shape from the first chunk - if self.__maxshape is None and self.__next_chunk.data is not None: - data_shape = get_data_shape(self.__next_chunk.data) - self.__maxshape = list(data_shape) - try: - self.__maxshape[0] = len(self.data) # We use self.data here because self.__data_iter does not allow len - except TypeError: - self.__maxshape[0] = None - self.__maxshape = tuple(self.__maxshape) - - # Determine the type of the data if possible - if self.__next_chunk.data is not None: - self.__dtype = self.__next_chunk.data.dtype - self.__first_chunk_shape = get_data_shape(self.__next_chunk.data) - - @classmethod - @docval({'name': 'data', 'type': None, 'doc': 'The data object used for iteration', 'default': None}, - {'name': 'maxshape', 'type': tuple, - 'doc': 'The maximum shape of the full data array. Use None to indicate unlimited dimensions', - 'default': None}, - {'name': 'dtype', 'type': np.dtype, 'doc': 'The Numpy data type for the array', 'default': None}, - {'name': 'buffer_size', 'type': int, 'doc': 'Number of values to be buffered in a chunk', 'default': 1}, - ) - def from_iterable(cls, **kwargs): - return cls(**kwargs) - - def __iter__(self): - """Return the iterator object""" - return self - - def _read_next_chunk(self): - """Read a single chunk from self.__data_iter and store the results in - self.__next_chunk - - :returns: self.__next_chunk, i.e., the DataChunk object describing the next chunk - """ - if self.__data_iter is not None: - curr_next_chunk = [] - for i in range(self.buffer_size): - try: - curr_next_chunk.append(next(self.__data_iter)) - except StopIteration: - pass - next_chunk_size = len(curr_next_chunk) - if next_chunk_size == 0: - self.__next_chunk = DataChunk(None, None) - else: - self.__next_chunk.data = np.asarray(curr_next_chunk) - if self.__next_chunk.selection is None: - self.__next_chunk.selection = slice(0, next_chunk_size) - else: - self.__next_chunk.selection = slice(self.__next_chunk.selection.stop, - self.__next_chunk.selection.stop+next_chunk_size) - else: - self.__next_chunk = DataChunk(None, None) - - return self.__next_chunk - - def __next__(self): - """Return the next data chunk or raise a StopIteration exception if all chunks have been retrieved. - - HINT: numpy.s_ provides a convenient way to generate index tuples using standard array slicing. This - is often useful to define the DataChunkk.selection of the current chunk - - :returns: DataChunk object with the data and selection of the current chunk - :rtype: DataChunk - - """ - # If we have not already read the next chunk, then read it now - if self.__next_chunk.data is None: - self._read_next_chunk() - # If we do not have any next chunk - if self.__next_chunk.data is None: - raise StopIteration - # If this is the first time we see a chunk then remember the size of the first chunk - if self.__first_chunk_shape is None: - self.__first_chunk_shape = self.__next_chunk.data.shape - # Keep the next chunk we need to return - curr_chunk = DataChunk(self.__next_chunk.data, - self.__next_chunk.selection) - # Remove the data for the next chunk from our list since we are returning it here. - # This is to allow the GarbageCollector to remmove the data when it goes out of scope and avoid - # having 2 full chunks in memory if not necessary - self.__next_chunk.data = None - # Return the current next chunk - return curr_chunk - - next = __next__ - - @docval(returns='Tuple with the recommended chunk shape or None if no particular shape is recommended.') - def recommended_chunk_shape(self): - """Recommend a chunk shape. - - To optimize iterative write the chunk should be aligned with the common shape of chunks returned by __next__ - or if those chunks are too large, then a well-aligned subset of those chunks. This may also be - any other value in case one wants to recommend chunk shapes to optimize read rather - than write. The default implementation returns None, indicating no preferential chunking option.""" - return None - - @docval(returns='Recommended initial shape for the full data. This should be the shape of the full dataset' + - 'if known beforehand or alternatively the minimum shape of the dataset. Return None if no ' + - 'recommendation is available') - def recommended_data_shape(self): - """Recommend an initial shape of the data. This is useful when progressively writing data and - we want to recommend and initial size for the dataset""" - if self.__maxshape is not None: - if np.all([i is not None for i in self.__maxshape]): - return self.__maxshape - return self.__first_chunk_shape - - @property - def maxshape(self): - return self.__maxshape - - @property - def dtype(self): - return self.__dtype - - -class DataChunk(object): - """ - Class used to describe a data chunk. Used in DataChunkIterator to describe - - :ivar data: Numpy ndarray with the data value(s) of the chunk - :ivar selection: Numpy index tuple describing the location of the chunk - """ - @docval({'name': 'data', 'type': np.ndarray, - 'doc': 'Numpy array with the data value(s) of the chunk', 'default': None}, - {'name': 'selection', 'type': None, - 'doc': 'Numpy index tuple describing the location of the chunk', 'default': None}) - def __init__(self, **kwargs): - self.data, self.selection = getargs('data', 'selection', kwargs) - - def __len__(self): - if self.data is not None: - return len(self.data) - else: - return 0 - - # Delegate attribute lookup to data object (See issue #636 - # https://github.com/NeurodataWithoutBorders/pynwb/issues/636#issuecomment-426742988) - def __getattr__(self, attr): - return getattr(self.data, attr) - - -def assertEqualShape(data1, - data2, - axes1=None, - axes2=None, - name1=None, - name2=None, - ignore_undetermined=True): - """ - Ensure that the shape of data1 and data2 match along the given dimensions - - :param data1: The first input array - :type data1: List, Tuple, np.ndarray, DataChunkIterator etc. - :param data2: The second input array - :type data2: List, Tuple, np.ndarray, DataChunkIterator etc. - :param name1: Optional string with the name of data1 - :param name2: Optional string with the name of data2 - :param axes1: The dimensions of data1 that should be matched to the dimensions of data2. Set to None to - compare all axes in order. - :type axes1: int, Tuple of ints, List of ints, or None - :param axes2: The dimensions of data2 that should be matched to the dimensions of data1. Must have - the same length as axes1. Set to None to compare all axes in order. - :type axes1: int, Tuple of ints, List of ints, or None - :param ignore_undetermined: Boolean indicating whether non-matching unlimited dimensions should be ignored, - i.e., if two dimension don't match because we can't determine the shape of either one, then - should we ignore that case or treat it as no match - - :return: Bool indicating whether the check passed and a string with a message about the matching process - """ - # Create the base return object - response = ShapeValidatorResult() - # Determine the shape of the datasets - response.shape1 = get_data_shape(data1) - response.shape2 = get_data_shape(data2) - # Determine the number of dimensions of the datasets - num_dims_1 = len(response.shape1) if response.shape1 is not None else None - num_dims_2 = len(response.shape2) if response.shape2 is not None else None - # Determine the string names of the datasets - n1 = name1 if name1 is not None else ("data1 at " + str(hex(id(data1)))) - n2 = name2 if name2 is not None else ("data2 at " + str(hex(id(data2)))) - # Determine the axes we should compare - response.axes1 = list(range(num_dims_1)) if axes1 is None else ([axes1] if isinstance(axes1, int) else axes1) - response.axes2 = list(range(num_dims_2)) if axes2 is None else ([axes2] if isinstance(axes2, int) else axes2) - # Validate the array shape - # 1) Check the number of dimensions of the arrays - if (response.axes1 is None and response.axes2 is None) and num_dims_1 != num_dims_2: - response.result = False - response.error = 'NUM_DIMS_ERROR' - response.message = response.SHAPE_ERROR[response.error] - response.message += " %s is %sD and %s is %sD" % (n1, num_dims_1, n2, num_dims_2) - # 2) Check that we have the same number of dimensions to compare on both arrays - elif len(response.axes1) != len(response.axes2): - response.result = False - response.error = 'NUM_AXES_ERROR' - response.message = response.SHAPE_ERROR[response.error] - response.message += " Cannot compare axes %s with %s" % (str(response.axes1), str(response.axes2)) - # 3) Check that the datasets have sufficient numner of dimensions - elif np.max(response.axes1) >= num_dims_1 or np.max(response.axes2) >= num_dims_2: - response.result = False - response.error = 'AXIS_OUT_OF_BOUNDS' - response.message = response.SHAPE_ERROR[response.error] - if np.max(response.axes1) >= num_dims_1: - response.message += "Insufficient number of dimensions for %s -- Expected %i found %i" % \ - (n1, np.max(response.axes1)+1, num_dims_1) - elif np.max(response.axes2) >= num_dims_2: - response.message += "Insufficient number of dimensions for %s -- Expected %i found %i" % \ - (n2, np.max(response.axes2)+1, num_dims_2) - # 4) Compare the length of the dimensions we should validate - else: - unmatched = [] - ignored = [] - for ax in zip(response.axes1, response.axes2): - if response.shape1[ax[0]] != response.shape2[ax[1]]: - if ignore_undetermined and (response.shape1[ax[0]] is None or response.shape2[ax[1]] is None): - ignored.append(ax) - else: - unmatched.append(ax) - response.unmatched = unmatched - response.ignored = ignored - - # Check if everything checked out - if len(response.unmatched) == 0: - response.result = True - response.error = None - response.message = response.SHAPE_ERROR[response.error] - if len(response.ignored) > 0: - response.message += " Ignored undetermined axes %s" % str(response.ignored) - else: - response.result = False - response.error = 'AXIS_LEN_ERROR' - response.message = response.SHAPE_ERROR[response.error] - response.message += "Axes %s with size %s of %s did not match dimensions %s with sizes %s of %s." % \ - (str([un[0] for un in response.unmatched]), - str([response.shape1[un[0]] for un in response.unmatched]), - n1, - str([un[1] for un in response.unmatched]), - str([response.shape2[un[1]] for un in response.unmatched]), - n2) - if len(response.ignored) > 0: - response.message += " Ignored undetermined axes %s" % str(response.ignored) - return response - - -class ShapeValidatorResult(object): - """Class for storing results from validating the shape of multi-dimensional arrays. - - This class is used to store results generated by ShapeValidator - - :ivar result: Boolean indicating whether results matched or not - :type result: bool - :ivar message: Message indicating the result of the matching procedure - :type messaage: str, None - """ - SHAPE_ERROR = {None: 'All required axes matched', - 'NUM_DIMS_ERROR': 'Unequal number of dimensions.', - 'NUM_AXES_ERROR': "Unequal number of axes for comparison.", - 'AXIS_OUT_OF_BOUNDS': "Axis index for comparison out of bounds.", - 'AXIS_LEN_ERROR': "Unequal length of axes."} - """ - Dict where the Keys are the type of errors that may have occurred during shape comparison and the - values are strings with default error messages for the type. - """ - - @docval({'name': 'result', 'type': bool, 'doc': 'Result of the shape validation', 'default': False}, - {'name': 'message', 'type': str, - 'doc': 'Message describing the result of the shape validation', 'default': None}, - {'name': 'ignored', 'type': tuple, - 'doc': 'Axes that have been ignored in the validaton process', 'default': tuple(), 'shape': (None, )}, - {'name': 'unmatched', 'type': tuple, - 'doc': 'List of axes that did not match during shape validation', 'default': tuple(), 'shape': (None, )}, - {'name': 'error', 'type': str, 'doc': 'Error that may have occurred. One of ERROR_TYPE', 'default': None}, - {'name': 'shape1', 'type': tuple, - 'doc': 'Shape of the first array for comparison', 'default': tuple(), 'shape': (None, )}, - {'name': 'shape2', 'type': tuple, - 'doc': 'Shape of the second array for comparison', 'default': tuple(), 'shape': (None, )}, - {'name': 'axes1', 'type': tuple, - 'doc': 'Axes for the first array that should match', 'default': tuple(), 'shape': (None, )}, - {'name': 'axes2', 'type': tuple, - 'doc': 'Axes for the second array that should match', 'default': tuple(), 'shape': (None, )}, - ) - def __init__(self, **kwargs): - self.result, self.message, self.ignored, self.unmatched, \ - self.error, self.shape1, self.shape2, self.axes1, self.axes2 = getargs( - 'result', 'message', 'ignored', 'unmatched', 'error', 'shape1', 'shape2', 'axes1', 'axes2', kwargs) - - def __setattr__(self, key, value): - """ - Overwrite to ensure that, e.g., error_message is not set to an illegal value. - """ - if key == 'error': - if value not in self.SHAPE_ERROR.keys(): - raise ValueError("Illegal error type. Error must be one of ShapeValidatorResult.SHAPE_ERROR: %s" - % str(self.SHAPE_ERROR)) - else: - super(ShapeValidatorResult, self).__setattr__(key, value) - elif key in ['shape1', 'shape2', 'axes1', 'axes2', 'ignored', 'unmatched']: # Make sure we sore tuples - super(ShapeValidatorResult, self).__setattr__(key, tuple(value)) - else: - super(ShapeValidatorResult, self).__setattr__(key, value) - - def __getattr__(self, item): - """ - Overwrite to allow dynamic retrival of the default message - """ - if item == 'default_message': - return self.SHAPE_ERROR[self.error] - return self.__getattribute__(item) - - -@docval_macro('data') -class DataIO(with_metaclass(ABCMeta, object)): - - @docval({'name': 'data', 'type': 'array_data', 'doc': 'the data to be written'}) - def __init__(self, **kwargs): - data = popargs('data', kwargs) - self.__data = data - - @property - def data(self): - return self.__data - - def __len__(self): - return len(self.__data) - - # Delegate attribute lookup to data object (See issue #636 - # https://github.com/NeurodataWithoutBorders/pynwb/issues/636#issuecomment-426742988) - def __getattr__(self, attr): - return getattr(self.data, attr) - - # Delegate iteration interface to data object: - def __next__(self): - return self.data.__next__() - - # Delegate iteration interface to data object: - def __iter__(self): - return self.data.__iter__() - - -class RegionSlicer(with_metaclass(ABCMeta, DataRegion)): - ''' - A abstract base class to control getting using a region - - Subclasses must implement `__getitem__` and `__len__` - ''' - - @docval({'name': 'target', 'type': None, 'doc': 'the target to slice'}, - {'name': 'slice', 'type': None, 'doc': 'the region to slice'}) - def __init__(self, **kwargs): - self.__target = getargs('target', kwargs) - self.__slice = getargs('slice', kwargs) - - @property - def data(self): - return self.target - - @property - def region(self): - return self.slice - - @property - def target(self): - return self.__target - - @property - def slice(self): - return self.__slice - - @abstractproperty - def __getitem__(self, idx): - pass - - @abstractproperty - def __len__(self): - pass - - -class ListSlicer(RegionSlicer): - - @docval({'name': 'dataset', 'type': (list, tuple, Data), 'doc': 'the HDF5 dataset to slice'}, - {'name': 'region', 'type': (list, tuple, slice), 'doc': 'the region reference to use to slice'}) - def __init__(self, **kwargs): - self.__dataset, self.__region = getargs('dataset', 'region', kwargs) - super(ListSlicer, self).__init__(self.__dataset, self.__region) - if isinstance(self.__region, slice): - self.__getter = itemgetter(self.__region) - self.__len = len(range(*self.__region.indices(len(self.__dataset)))) - else: - self.__getter = itemgetter(*self.__region) - self.__len = len(self.__region) - - def __read_region(self): - if not hasattr(self, '_read'): - self._read = self.__getter(self.__dataset) - del self.__getter - - def __getitem__(self, idx): - self.__read_region() - getter = None - if isinstance(idx, (list, tuple)): - getter = itemgetter(*idx) - else: - getter = itemgetter(idx) - return getter(self._read) - - def __len__(self): - return self.__len diff --git a/src/pynwb/form/monitor.py b/src/pynwb/form/monitor.py deleted file mode 100644 index 4d2d2a0f6..000000000 --- a/src/pynwb/form/monitor.py +++ /dev/null @@ -1,76 +0,0 @@ -from abc import ABCMeta, abstractmethod -import six - -from .utils import docval -from .data_utils import AbstractDataChunkIterator, DataChunkIterator, DataChunk - - -class NotYetExhausted(Exception): - pass - - -@six.add_metaclass(ABCMeta) -class DataChunkProcessor(AbstractDataChunkIterator): - - @docval({'name': 'data', 'type': DataChunkIterator, 'doc': 'the DataChunkIterator to analyze'}) - def __init__(self, **kwargs): - """Initialize the DataChunkIterator""" - # Get the user parameters - self.__dci = getargs('data', kwargs) # noqa: F821 - - def __next__(self): - try: - dc = self.__dci.__next__() - except StopIteration as e: - self.__done = True - raise e - self.process_data_chunk(dc) - return dc - - def __iter__(self): - return iter(self.__dci) - - def recommended_chunk_shape(self): - return self.__dci.recommended_chunk_shape() - - def recommended_data_shape(self): - return self.__dci.recommended_data_shape() - - def get_final_result(self, **kwargs): - ''' Return the result of processing data fed by this DataChunkIterator ''' - if not self.__done: - raise NotYetExhausted() - return self.compute_final_result() - - @abstractmethod - @docval({'name': 'data_chunk', 'type': DataChunk, 'doc': 'a chunk to process'}) - def process_data_chunk(self, **kwargs): - ''' This method should take in a DataChunk, - and process it. - ''' - pass - - @abstractmethod - @docval(returns='the result of processing this stream') - def compute_final_result(self, **kwargs): - ''' Return the result of processing this stream - Should raise NotYetExhaused exception - ''' - pass - - -class NumSampleCounter(DataChunkProcessor): - - def __init__(self, **kwargs): - args, kwargs = fmt_docval_args(DataChunkProcessor.__init__, kwargs) # noqa: F821 - super(NumSampleCounter, self).__init__(*args, **kwargs) - self.__sample_count = 0 - - @docval({'name': 'data_chunk', 'type': DataChunk, 'doc': 'a chunk to process'}) - def process_data_chunk(self, **kwargs): - dc = getargs('data_chunk', kwargs) # noqa: F821 - self.__sample_count += len(dc) - - @docval(returns='the result of processing this stream') - def compute_final_result(self, **kwargs): - return self.__sample_count diff --git a/src/pynwb/form/query.py b/src/pynwb/form/query.py deleted file mode 100644 index a309aeced..000000000 --- a/src/pynwb/form/query.py +++ /dev/null @@ -1,162 +0,0 @@ -from six import with_metaclass -import numpy as np - -from .utils import ExtenderMeta, docval_macro, docval, getargs -from .array import Array - - -class Query(with_metaclass(ExtenderMeta, object)): - - __operations__ = ( - '__lt__', - '__gt__', - '__le__', - '__ge__', - '__eq__', - '__ne__', - ) - - @classmethod - def __build_operation(cls, op): - def __func(self, arg): - return cls(self, op, arg) - - @ExtenderMeta.pre_init - def __make_operators(cls, name, bases, classdict): - if not isinstance(cls.__operations__, tuple): - raise TypeError("'__operations__' must be of type tuple") - # add any new operations - if len(bases) and 'Query' in globals() and issubclass(bases[-1], Query) \ - and bases[-1].__operations__ is not cls.__operations__: - new_operations = list(cls.__operations__) - new_operations[0:0] = bases[-1].__operations__ - cls.__operations__ = tuple(new_operations) - for op in cls.__operations__: - if not hasattr(cls, op): - setattr(cls, op, cls.__build_operation(op)) - - def __init__(self, obj, op, arg): - self.obj = obj - self.op = op - self.arg = arg - self.collapsed = None - self.expanded = None - - @docval({'name': 'expand', 'type': bool, 'help': 'whether or not to expand result', 'default': True}) - def evaluate(self, **kwargs): - expand = getargs('expand', kwargs) - if expand: - if self.expanded is None: - self.expanded = self.__evalhelper() - return self.expanded - else: - if self.collapsed is None: - self.collapsed = self.__collapse(self.__evalhelper()) - return self.collapsed - - def __evalhelper(self): - obj = self.obj - arg = self.arg - if isinstance(obj, Query): - obj = obj.evaluate() - elif isinstance(obj, FORMDataset): - obj = obj.dataset - if isinstance(arg, Query): - arg = self.arg.evaluate() - return getattr(obj, self.op)(self.arg) - - def __collapse(self, result): - if isinstance(result, slice): - return (result.start, result.stop) - elif isinstance(result, list): - ret = list() - for idx in result: - if isinstance(idx, slice) and (idx.step is None or idx.step == 1): - ret.append((idx.start, idx.stop)) - else: - ret.append(idx) - return ret - else: - return result - - def __and__(self, other): - return NotImplemented - - def __or__(self, other): - return NotImplemented - - def __xor__(self, other): - return NotImplemented - - def __contains__(self, other): - return NotImplemented - - -@docval_macro('array_data') -class FORMDataset(with_metaclass(ExtenderMeta, object)): - - __operations__ = ( - '__lt__', - '__gt__', - '__le__', - '__ge__', - '__eq__', - '__ne__', - ) - - @classmethod - def __build_operation(cls, op): - def __func(self, arg): - return Query(self, op, arg) - setattr(__func, '__name__', op) - return __func - - @ExtenderMeta.pre_init - def __make_operators(cls, name, bases, classdict): - if not isinstance(cls.__operations__, tuple): - raise TypeError("'__operations__' must be of type tuple") - # add any new operations - if len(bases) and 'Query' in globals() and issubclass(bases[-1], Query) \ - and bases[-1].__operations__ is not cls.__operations__: - new_operations = list(cls.__operations__) - new_operations[0:0] = bases[-1].__operations__ - cls.__operations__ = tuple(new_operations) - for op in cls.__operations__: - setattr(cls, op, cls.__build_operation(op)) - - def __evaluate_key(self, key): - if isinstance(key, (tuple, list, np.ndarray)): - return tuple(map(self.__evaluate_key, key)) - else: - if isinstance(key, Query): - return key.evaluate() - return key - - def __getitem__(self, key): - idx = self.__evaluate_key(key) - return self.dataset[idx] - - @docval({'name': 'dataset', 'type': ('array_data', Array), 'doc': 'the HDF5 file lazily evaluate'}) - def __init__(self, **kwargs): - super(FORMDataset, self).__init__() - self.__dataset = getargs('dataset', kwargs) - - @property - def dataset(self): - return self.__dataset - - @property - def dtype(self): - return self.__dataset.dtype - - def __len__(self): - return len(self.__dataset) - - def __iter__(self): - return iter(self.dataset) - - def __next__(self): - return next(self.dataset) - - def next(self): - return self.dataset.next() diff --git a/src/pynwb/form/spec/__init__.py b/src/pynwb/form/spec/__init__.py deleted file mode 100644 index 88dc2edc6..000000000 --- a/src/pynwb/form/spec/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# flake8: noqa: F401 -import os.path - -from .spec import NAME_WILDCARD -from .spec import Spec -from .spec import AttributeSpec -from .spec import DtypeSpec -from .spec import DtypeHelper -from .spec import RefSpec -from .spec import DatasetSpec -from .spec import LinkSpec -from .spec import GroupSpec -from .catalog import SpecCatalog -from .namespace import SpecNamespace -from .namespace import NamespaceCatalog -from .namespace import SpecReader -from .write import NamespaceBuilder -from .write import SpecWriter - -from ..utils import docval diff --git a/src/pynwb/form/spec/catalog.py b/src/pynwb/form/spec/catalog.py deleted file mode 100644 index 6db425cac..000000000 --- a/src/pynwb/form/spec/catalog.py +++ /dev/null @@ -1,132 +0,0 @@ -from collections import OrderedDict - -from .spec import BaseStorageSpec, GroupSpec -from ..utils import docval, getargs - - -class SpecCatalog(object): - - def __init__(self): - ''' - Create a new catalog for storing specifications - - ** Private Instance Variables ** - - :ivar __specs: Dict with the specification of each registered type - :ivar __parent_types: Dict with parent types for each registered type - :ivar __spec_source_files: Dict with the path to the source files (if available) for each registered type - :ivar __hierarchy: Dict describing the hierarchy for each registered type. - NOTE: Always use SpecCatalog.get_hierarchy(...) to retrieve the hierarchy - as this dictionary is used like a cache, i.e., to avoid repeated calcuation - of the hierarchy but the contents are computed on first request by SpecCatalog.get_hierarchy(...) - ''' - self.__specs = OrderedDict() - self.__parent_types = dict() - self.__hierarchy = dict() - self.__spec_source_files = dict() - - @docval({'name': 'spec', 'type': BaseStorageSpec, 'doc': 'a Spec object'}, - {'name': 'source_file', 'type': str, - 'doc': 'path to the source file from which the spec was loaded', 'default': None}) - def register_spec(self, **kwargs): - ''' - Associate a specified object type with an HDF5 specification - ''' - spec, source_file = getargs('spec', 'source_file', kwargs) - ndt = spec.data_type_inc - ndt_def = spec.data_type_def - if ndt_def is None: - raise ValueError('cannot register spec that has no data_type_def') - if ndt_def != ndt: - self.__parent_types[ndt_def] = ndt - type_name = ndt_def if ndt_def is not None else ndt - if type_name in self.__specs: - raise ValueError("'%s' - cannot overwrite existing specification" % type_name) - self.__specs[type_name] = spec - self.__spec_source_files[type_name] = source_file - - @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the Spec for'}, - returns="the specification for writing the given object type to HDF5 ", rtype='Spec') - def get_spec(self, **kwargs): - ''' - Get the Spec object for the given type - ''' - data_type = getargs('data_type', kwargs) - return self.__specs.get(data_type, None) - - @docval(rtype=tuple) - def get_registered_types(self, **kwargs): - ''' - Return all registered specifications - ''' - return tuple(self.__specs.keys()) - - @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type of the spec to get the source file for'}, - returns="the path to source specification file from which the spec was originally loaded or None ", - rtype='str') - def get_spec_source_file(self, **kwargs): - ''' - Return the path to the source file from which the spec for the given - type was loaded from. None is returned if no file path is available - for the spec. Note: The spec in the file may not be identical to the - object in case the spec is modified after load. - ''' - data_type = getargs('data_type', kwargs) - return self.__spec_source_files.get(data_type, None) - - @docval({'name': 'spec', 'type': BaseStorageSpec, 'doc': 'the Spec object to register'}, - {'name': 'source_file', - 'type': str, - 'doc': 'path to the source file from which the spec was loaded', 'default': None}, - rtype=tuple, returns='the types that were registered with this spec') - def auto_register(self, **kwargs): - ''' - Register this specification and all sub-specification using data_type as object type name - ''' - spec, source_file = getargs('spec', 'source_file', kwargs) - ndt = spec.data_type_def - ret = list() - if ndt is not None: - self.register_spec(spec, source_file) - ret.append(ndt) - if isinstance(spec, GroupSpec): - for dataset_spec in spec.datasets: - dset_ndt = dataset_spec.data_type_def - if dset_ndt is not None and not spec.is_inherited_type(dataset_spec): - ret.append(dset_ndt) - self.register_spec(dataset_spec, source_file) - for group_spec in spec.groups: - ret.extend(self.auto_register(group_spec, source_file)) - return tuple(ret) - - @docval({'name': 'data_type', 'type': (str, type), - 'doc': 'the data_type to get the hierarchy of'}) - def get_hierarchy(self, **kwargs): - ''' Get the extension hierarchy for the given data_type ''' - data_type = getargs('data_type', kwargs) - if isinstance(data_type, type): - data_type = data_type.__name__ - ret = self.__hierarchy.get(data_type) - if ret is None: - hierarchy = list() - parent = data_type - while parent is not None: - hierarchy.append(parent) - parent = self.__parent_types.get(parent) - # store computed hierarchy for later - tmp_hier = tuple(hierarchy) - ret = tmp_hier - while len(tmp_hier) > 0: - self.__hierarchy[tmp_hier[0]] = tmp_hier - tmp_hier = tmp_hier[1:] - return tuple(ret) - - def __copy__(self): - ret = SpecCatalog() - ret.__specs = copy.copy(spec) # noqa: F821 - return ret - - def __deepcopy__(self): - ret = SpecCatalog() - ret.__specs = copy.deepcopy(spec) # noqa: F821 - return ret diff --git a/src/pynwb/form/spec/namespace.py b/src/pynwb/form/spec/namespace.py deleted file mode 100644 index 411f8be1a..000000000 --- a/src/pynwb/form/spec/namespace.py +++ /dev/null @@ -1,432 +0,0 @@ -from collections import OrderedDict -from datetime import datetime -from copy import deepcopy, copy -import ruamel.yaml as yaml -import os.path -import string -from warnings import warn -from itertools import chain -from abc import ABCMeta, abstractmethod -from six import with_metaclass, raise_from - - -from ..utils import docval, getargs, popargs, get_docval, call_docval_func -from .catalog import SpecCatalog -from .spec import DatasetSpec, GroupSpec - - -_namespace_args = [ - {'name': 'doc', 'type': str, 'doc': 'a description about what this namespace represents'}, - {'name': 'name', 'type': str, 'doc': 'the name of this namespace'}, - {'name': 'schema', 'type': list, 'doc': 'location of schema specification files or other Namespaces'}, - {'name': 'full_name', 'type': str, 'doc': 'extended full name of this namespace', 'default': None}, - {'name': 'version', 'type': (str, tuple, list), 'doc': 'Version number of the namespace', 'default': None}, - {'name': 'date', 'type': (datetime, str), - 'doc': "Date last modified or released. Formatting is %Y-%m-%d %H:%M:%S, e.g, 2017-04-25 17:14:13", - 'default': None}, - {'name': 'author', 'type': (str, list), 'doc': 'Author or list of authors.', 'default': None}, - {'name': 'contact', 'type': (str, list), - 'doc': 'List of emails. Ordering should be the same as for author', 'default': None}, - {'name': 'catalog', 'type': SpecCatalog, 'doc': 'The SpecCatalog object for this SpecNamespace', 'default': None} -] - - -class SpecNamespace(dict): - """ - A namespace for specifications - """ - - __types_key = 'data_types' - - @docval(*deepcopy(_namespace_args)) - def __init__(self, **kwargs): - doc, full_name, name, version, date, author, contact, schema, catalog = \ - popargs('doc', 'full_name', 'name', 'version', 'date', 'author', 'contact', 'schema', 'catalog', kwargs) - super(SpecNamespace, self).__init__() - self['doc'] = doc - self['schema'] = schema - if any(c in string.whitespace for c in name): - raise ValueError("'name' must not contain any whitespace") - self['name'] = name - if full_name is not None: - self['full_name'] = full_name - if version is not None: - self['version'] = version - if date is not None: - self['date'] = date - if author is not None: - self['author'] = author - if contact is not None: - self['contact'] = contact - self.__catalog = catalog if catalog is not None else SpecCatalog() - - @classmethod - def types_key(cls): - ''' Get the key used for specifying types to include from a file or namespace - - Override this method to use a different name for 'data_types' - ''' - return cls.__types_key - - @property - def full_name(self): - """String with full name or None""" - return self.get('full_name', None) - - @property - def contact(self): - """String or list of strings with the contacts or None""" - return self.get('contact', None) - - @property - def author(self): - """String or list of strings with the authors or None""" - return self.get('author', None) - - @property - def version(self): - """String, list, or tuple with the version or None """ - return self.get('version', None) - - @property - def date(self): - """Date last modified or released. - - :return: datetime object, string, or None""" - return self.get('full_name', None) - - @property - def name(self): - """String with short name or None""" - return self.get('name', None) - - @property - def doc(self): - return self['doc'] - - @property - def schema(self): - return self['schema'] - - @property - def catalog(self): - """The SpecCatalog containing all the Specs""" - return self.__catalog - - @docval({'name': 'data_type', 'type': (str, type), 'doc': 'the data_type to get the spec for'}) - def get_spec(self, **kwargs): - """Get the Spec object for the given data type""" - data_type = getargs('data_type', kwargs) - spec = self.__catalog.get_spec(data_type) - if spec is None: - raise ValueError("No specification for '%s' in namespace '%s'" % (data_type, self.name)) - return spec - - @docval(returns="the a tuple of the available data types", rtype=tuple) - def get_registered_types(self, **kwargs): - """Get the available types in this namespace""" - return self.__catalog.get_registered_types() - - @docval({'name': 'data_type', 'type': (str, type), 'doc': 'the data_type to get the hierarchy of'}, - returns="a tuple with the type hierarchy", rtype=tuple) - def get_hierarchy(self, **kwargs): - ''' Get the extension hierarchy for the given data_type in this namespace''' - data_type = getargs('data_type', kwargs) - return self.__catalog.get_hierarchy(data_type) - - @classmethod - def build_namespace(cls, **spec_dict): - kwargs = copy(spec_dict) - try: - args = [kwargs.pop(x['name']) for x in get_docval(cls.__init__) if 'default' not in x] - except KeyError as e: - raise KeyError("'%s' not found in %s" % (e.args[0], str(spec_dict))) - return cls(*args, **kwargs) - - -class SpecReader(with_metaclass(ABCMeta, object)): - - @docval({'name': 'source', 'type': str, 'doc': 'the source from which this reader reads from'}) - def __init__(self, **kwargs): - self.__source = getargs('source', kwargs) - - @property - def source(self): - return self.__source - - @abstractmethod - def read_spec(self): - pass - - @abstractmethod - def read_namespace(self): - pass - - -class YAMLSpecReader(SpecReader): - - @docval({'name': 'indir', 'type': str, 'doc': 'the path spec files are relative to', 'default': '.'}) - def __init__(self, **kwargs): - super_kwargs = {'source': kwargs['indir']} - call_docval_func(super(YAMLSpecReader, self).__init__, super_kwargs) - - def read_namespace(self, namespace_path): - namespaces = None - with open(namespace_path, 'r') as stream: - d = yaml.safe_load(stream) - namespaces = d.get('namespaces') - if namespaces is None: - raise ValueError("no 'namespaces' found in %s" % namespace_path) - return namespaces - - def read_spec(self, spec_path): - specs = None - with open(self.__get_spec_path(spec_path), 'r') as stream: - specs = yaml.safe_load(stream) - if not ('datasets' in specs or 'groups' in specs): - raise ValueError("no 'groups' or 'datasets' found in %s" % spec_path) - return specs - - def __get_spec_path(self, spec_path): - if os.path.isabs(spec_path): - return spec_path - return os.path.join(self.source, spec_path) - - -class NamespaceCatalog(object): - - @docval({'name': 'group_spec_cls', 'type': type, - 'doc': 'the class to use for group specifications', 'default': GroupSpec}, - {'name': 'dataset_spec_cls', 'type': type, - 'doc': 'the class to use for dataset specifications', 'default': DatasetSpec}, - {'name': 'spec_namespace_cls', 'type': type, - 'doc': 'the class to use for specification namespaces', 'default': SpecNamespace},) - def __init__(self, **kwargs): - """Create a catalog for storing multiple Namespaces""" - self.__namespaces = OrderedDict() - self.__dataset_spec_cls = getargs('dataset_spec_cls', kwargs) - self.__group_spec_cls = getargs('group_spec_cls', kwargs) - self.__spec_namespace_cls = getargs('spec_namespace_cls', kwargs) - # keep track of all spec objects ever loaded, so we don't have - # multiple object instances of a spec - self.__loaded_specs = dict() - self.__included_specs = dict() - self.__included_sources = dict() - - def __copy__(self): - ret = NamespaceCatalog(self.__group_spec_cls, - self.__dataset_spec_cls, - self.__spec_namespace_cls) - ret.__namespaces = copy(self.__namespaces) - ret.__loaded_specs = copy(self.__loaded_specs) - ret.__included_specs = copy(self.__included_specs) - ret.__included_sources = copy(self.__included_sources) - return ret - - @property - @docval(returns='a tuple of the available namespaces', rtype=tuple) - def namespaces(self): - """The namespaces in this NamespaceCatalog""" - return tuple(self.__namespaces.keys()) - - @property - def dataset_spec_cls(self): - """The DatasetSpec class used in this NamespaceCatalog""" - return self.__dataset_spec_cls - - @property - def group_spec_cls(self): - """The GroupSpec class used in this NamespaceCatalog""" - return self.__group_spec_cls - - @property - def spec_namespace_cls(self): - """The SpecNamespace class used in this NamespaceCatalog""" - return self.__spec_namespace_cls - - @docval({'name': 'name', 'type': str, 'doc': 'the name of this namespace'}, - {'name': 'namespace', 'type': SpecNamespace, 'doc': 'the SpecNamespace object'}) - def add_namespace(self, **kwargs): - """Add a namespace to this catalog""" - name, namespace = getargs('name', 'namespace', kwargs) - if name in self.__namespaces: - raise KeyError("namespace '%s' already exists" % name) - self.__namespaces[name] = namespace - - @docval({'name': 'name', 'type': str, 'doc': 'the name of this namespace'}, - returns="the SpecNamespace with the given name", rtype=SpecNamespace) - def get_namespace(self, **kwargs): - """Get the a SpecNamespace""" - name = getargs('name', kwargs) - ret = self.__namespaces.get(name) - if ret is None: - raise KeyError("'%s' not a namespace" % name) - return ret - - @docval({'name': 'namespace', 'type': str, 'doc': 'the name of the namespace'}, - {'name': 'data_type', 'type': (str, type), 'doc': 'the data_type to get the spec for'}, - returns="the specification for writing the given object type to HDF5 ", rtype='Spec') - def get_spec(self, **kwargs): - ''' - Get the Spec object for the given type from the given Namespace - ''' - namespace, data_type = getargs('namespace', 'data_type', kwargs) - if namespace not in self.__namespaces: - raise KeyError("'%s' not a namespace" % namespace) - return self.__namespaces[namespace].get_spec(data_type) - - @docval({'name': 'namespace', 'type': str, 'doc': 'the name of the namespace'}, - {'name': 'data_type', 'type': (str, type), 'doc': 'the data_type to get the spec for'}, - returns="a tuple with the type hierarchy", rtype=tuple) - def get_hierarchy(self, **kwargs): - ''' - Get the type hierarchy for a given data_type in a given namespace - ''' - namespace, data_type = getargs('namespace', 'data_type', kwargs) - spec_ns = self.__namespaces.get(namespace) - if spec_ns is None: - raise KeyError("'%s' not a namespace" % namespace) - return spec_ns.get_hierarchy(data_type) - - @docval(rtype=tuple) - def get_sources(self, **kwargs): - ''' - Get all the source specification files that were loaded in this catalog - ''' - return tuple(self.__loaded_specs.keys()) - - @docval({'name': 'namespace', 'type': str, 'doc': 'the name of the namespace'}, - rtype=tuple) - def get_namespace_sources(self, **kwargs): - ''' - Get all the source specifications that were loaded for a given namespace - ''' - namespace = getargs('namespace', kwargs) - return tuple(self.__included_sources[namespace]) - - @docval({'name': 'source', 'type': str, 'doc': 'the name of the source'}, - rtype=tuple) - def get_types(self, **kwargs): - ''' - Get the types that were loaded from a given source - ''' - source = getargs('source', kwargs) - ret = self.__loaded_specs.get(source) - if ret is not None: - ret = tuple(ret) - return ret - - def __load_spec_file(self, reader, spec_source, catalog, dtypes=None, resolve=True): - ret = self.__loaded_specs.get(spec_source) - - def __reg_spec(spec_cls, spec_dict): - dt_def = spec_dict.get(spec_cls.def_key()) - if dt_def is None: - msg = 'skipping spec in %s, no %s found' % (spec_source, spec_cls.def_key()) - warn(msg) - return - if dtypes and dt_def not in dtypes: - return - if resolve: - self.__resolve_includes(spec_dict, catalog) - spec_obj = spec_cls.build_spec(spec_dict) - return catalog.auto_register(spec_obj, spec_source) - - if ret is None: - ret = list() - d = reader.read_spec(spec_source) - specs = d.get('datasets', list()) - for spec_dict in specs: - ret.extend(__reg_spec(self.__dataset_spec_cls, spec_dict)) - specs = d.get('groups', list()) - for spec_dict in specs: - ret.extend(__reg_spec(self.__group_spec_cls, spec_dict)) - self.__loaded_specs[spec_source] = ret - return ret - - def __resolve_includes(self, spec_dict, catalog): - """ - Pull in any attributes, datasets, or groups included - """ - dt_inc = spec_dict.get(self.__group_spec_cls.inc_key()) - dt_def = spec_dict.get(self.__group_spec_cls.def_key()) - if dt_inc is not None and dt_def is not None: - parent_spec = catalog.get_spec(dt_inc) - if parent_spec is None: - msg = "Cannot resolve include spec '%s' for type '%s'" % (dt_inc, dt_def) - raise ValueError(msg) - spec_dict[self.__group_spec_cls.inc_key()] = parent_spec - it = chain(spec_dict.get('groups', list()), spec_dict.get('datasets', list())) - for subspec_dict in it: - self.__resolve_includes(subspec_dict, catalog) - - def __load_namespace(self, namespace, reader, types_key, resolve=True): - ns_name = namespace['name'] - if ns_name in self.__namespaces: - raise KeyError("namespace '%s' already exists" % ns_name) - catalog = SpecCatalog() - included_types = dict() - for s in namespace['schema']: - if 'source' in s: - # read specs from file - dtypes = None - if types_key in s: - dtypes = set(s[types_key]) - self.__load_spec_file(reader, s['source'], catalog, dtypes=dtypes, resolve=resolve) - self.__included_sources.setdefault(ns_name, list()).append(s['source']) - elif 'namespace' in s: - # load specs from namespace - try: - inc_ns = self.get_namespace(s['namespace']) - except KeyError as e: - raise_from(ValueError("Could not load namespace '%s'" % s['namespace']), e) - if types_key in s: - types = s[types_key] - else: - types = inc_ns.get_registered_types() - for ndt in types: - spec = inc_ns.get_spec(ndt) - spec_file = inc_ns.catalog.get_spec_source_file(ndt) - catalog.register_spec(spec, spec_file) - included_types[s['namespace']] = tuple(types) - # construct namespace - self.add_namespace(ns_name, - self.__spec_namespace_cls.build_namespace(catalog=catalog, **namespace)) - return included_types - - @docval({'name': 'namespace_path', 'type': str, 'doc': 'the path to the file containing the namespaces(s) to load'}, - {'name': 'resolve', - 'type': bool, - 'doc': 'whether or not to include objects from included/parent spec objects', 'default': True}, - {'name': 'reader', - 'type': SpecReader, - 'doc': 'the class to user for reading specifications', 'default': None}, - returns='a dictionary describing the dependencies of loaded namespaces', rtype=dict) - def load_namespaces(self, **kwargs): - """Load the namespaces in the given file""" - namespace_path, resolve, reader = getargs('namespace_path', 'resolve', 'reader', kwargs) - if reader is None: - # load namespace definition from file - if not os.path.exists(namespace_path): - msg = "namespace file '%s' not found" % namespace_path - raise IOError(msg) - reader = YAMLSpecReader(indir=os.path.dirname(namespace_path)) - ns_path_key = os.path.join(reader.source, os.path.basename(namespace_path)) - ret = self.__included_specs.get(ns_path_key) - if ret is None: - ret = dict() - else: - return ret - namespaces = reader.read_namespace(namespace_path) - types_key = self.__spec_namespace_cls.types_key() - to_load = list() - for ns in namespaces: - if ns['name'] in self.__namespaces: - warn("ignoring namespace '%s' because it already exists" % ns['name']) - else: - to_load.append(ns) - # now load specs into namespace - for ns in to_load: - ret[ns['name']] = self.__load_namespace(ns, reader, types_key, resolve=resolve) - self.__included_specs[ns_path_key] = ret - return ret diff --git a/src/pynwb/form/spec/spec.py b/src/pynwb/form/spec/spec.py deleted file mode 100644 index 23f5072ae..000000000 --- a/src/pynwb/form/spec/spec.py +++ /dev/null @@ -1,1216 +0,0 @@ -import abc -from copy import deepcopy -from collections import OrderedDict -import re -from warnings import warn - -from ..utils import docval, getargs, popargs, get_docval, fmt_docval_args - -NAME_WILDCARD = None -ZERO_OR_ONE = '?' -ZERO_OR_MANY = '*' -ONE_OR_MANY = '+' -DEF_QUANTITY = 1 -FLAGS = { - 'zero_or_one': ZERO_OR_ONE, - 'zero_or_many': ZERO_OR_MANY, - 'one_or_many': ONE_OR_MANY -} - -from six import with_metaclass # noqa: E402 - - -class DtypeHelper(): - # Dict where the keys are the primary data type and the values are list of strings with synonyms for the dtype - primary_dtype_synonyms = { - 'float': ["float", "float32"], - 'double': ["double", "float64"], - 'short': ["int16", "short"], - 'int': ["int32", "int"], - 'long': ["int64", "long"], - 'utf': ["text", "utf", "utf8", "utf-8"], - 'ascii': ["ascii", "bytes"], - 'bool': ["bool"], - 'int8': ["int8"], - 'uint8': ["uint8"], - 'uint16': ["uint16"], - 'uint32': ["uint32", "uint"], - 'uint64': ["uint64"], - 'object': ['object'], - 'region': ['region'], - 'numeric': ['numeric'], - 'isodatetime': ["isodatetime", "datetime", "datetime64"] - } - - # List of recommended primary dtype strings. These are the keys of primary_dtype_string_synonyms - recommended_primary_dtypes = list(primary_dtype_synonyms.keys()) - - # List of valid primary data type strings - valid_primary_dtypes = set(list(primary_dtype_synonyms.keys()) + - [vi for v in primary_dtype_synonyms.values() for vi in v]) - - @staticmethod - def simplify_cpd_type(cpd_type): - ''' - Transform a list of DtypeSpecs into a list of strings. - Use for simple representation of compound type and - validation. - - :param cpd_type: The list of DtypeSpecs to simplify - :type cpd_type: list - - ''' - ret = list() - for exp in cpd_type: - exp_key = exp.dtype - if isinstance(exp_key, RefSpec): - exp_key = exp_key.reftype - ret.append(exp_key) - return ret - - -class ConstructableDict(with_metaclass(abc.ABCMeta, dict)): - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this ConstructableDict class from a dictionary ''' - return deepcopy(spec_dict) # noqa: F821 - - @classmethod - def build_spec(cls, spec_dict): - ''' Build a Spec object from the given Spec dict ''' - vargs = cls.build_const_args(spec_dict) - args = list() - kwargs = dict() - try: - - for x in get_docval(cls.__init__): - if not x['name'] in vargs: - continue - if 'default' not in x: - args.append(vargs.get(x['name'])) - else: - kwargs[x['name']] = vargs.get(x['name']) - except KeyError as e: - raise KeyError("'%s' not found in %s" % (e.args[0], str(spec_dict))) - return cls(*args, **kwargs) - - -class Spec(ConstructableDict): - ''' A base specification class - ''' - - @docval({'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - {'name': 'name', 'type': str, 'doc': 'The name of this attribute', 'default': None}, - {'name': 'required', 'type': bool, 'doc': 'whether or not this attribute is required', 'default': True}, - {'name': 'parent', 'type': 'Spec', 'doc': 'the parent of this spec', 'default': None}) - def __init__(self, **kwargs): - name, doc, required, parent = getargs('name', 'doc', 'required', 'parent', kwargs) - super(Spec, self).__init__() - if name is not None: - self['name'] = name - if doc is not None: - self['doc'] = doc - if not required: - self['required'] = required - self._parent = parent - - @property - def doc(self): - ''' Documentation on what this Spec is specifying ''' - return self.get('doc', None) - - @property - def name(self): - ''' The name of the object being specified ''' - return self.get('name', None) - - @property - def parent(self): - ''' The parent specification of this specification ''' - return self._parent - - @parent.setter - def parent(self, spec): - ''' Set the parent of this specification ''' - if self._parent is not None: - raise Exception('Cannot re-assign parent') - self._parent = spec - - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this Spec class from a dictionary ''' - ret = super(Spec, cls).build_const_args(spec_dict) - if 'doc' not in ret: - msg = "'doc' missing: %s" % str(spec_dict) - raise ValueError(msg) - return ret - - def __hash__(self): - return id(self) - -# def __eq__(self, other): -# return id(self) == id(other) - - -_target_type_key = 'target_type' - -_ref_args = [ - {'name': _target_type_key, 'type': str, 'doc': 'the target type GroupSpec or DatasetSpec'}, - {'name': 'reftype', 'type': str, 'doc': 'the type of references this is i.e. region or object'}, -] - - -class RefSpec(ConstructableDict): - - __allowable_types = ('object', 'region') - - @docval(*_ref_args) - def __init__(self, **kwargs): - target_type, reftype = getargs(_target_type_key, 'reftype', kwargs) - self[_target_type_key] = target_type - if reftype not in self.__allowable_types: - msg = "reftype must be one of the following: %s" % ", ".join(self.__allowable_types) - raise ValueError(msg) - self['reftype'] = reftype - - @property - def target_type(self): - '''The data_type of the target of the reference''' - return self[_target_type_key] - - @property - def reftype(self): - '''The type of reference''' - return self['reftype'] - - @docval(rtype=bool, returns='True if this RefSpec specifies a region reference, False otherwise') - def is_region(self): - return self['reftype'] == 'region' - - -_attr_args = [ - {'name': 'name', 'type': str, 'doc': 'The name of this attribute'}, - {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - {'name': 'dtype', 'type': (str, RefSpec), 'doc': 'The data type of this attribute'}, - {'name': 'shape', 'type': (list, tuple), 'doc': 'the shape of this dataset', 'default': None}, - {'name': 'dims', 'type': (list, tuple), 'doc': 'the dimensions of this dataset', 'default': None}, - {'name': 'required', 'type': bool, - 'doc': 'whether or not this attribute is required. ignored when "value" is specified', 'default': True}, - {'name': 'parent', 'type': 'BaseStorageSpec', 'doc': 'the parent of this spec', 'default': None}, - {'name': 'value', 'type': None, 'doc': 'a constant value for this attribute', 'default': None}, - {'name': 'default_value', 'type': None, 'doc': 'a default value for this attribute', 'default': None} -] - - -class AttributeSpec(Spec): - ''' Specification for attributes - ''' - - @docval(*_attr_args) - def __init__(self, **kwargs): - name, dtype, doc, dims, shape, required, parent, value, default_value = getargs( - 'name', 'dtype', 'doc', 'dims', 'shape', 'required', 'parent', 'value', 'default_value', kwargs) - super(AttributeSpec, self).__init__(doc, name=name, required=required, parent=parent) - if isinstance(dtype, RefSpec): - self['dtype'] = dtype - else: - self['dtype'] = dtype - # Validate the dype string - if self['dtype'] not in DtypeHelper.valid_primary_dtypes: - raise ValueError('dtype %s not a valid primary data type %s' % (self['dtype'], - str(DtypeHelper.valid_primary_dtypes))) - if value is not None: - self.pop('required', None) - self['value'] = value - if default_value is not None: - if value is not None: - raise ValueError("cannot specify 'value' and 'default_value'") - self['default_value'] = default_value - self['required'] = False - if shape is not None: - self['shape'] = shape - if dims is not None: - self['dims'] = dims - if 'shape' not in self: - self['shape'] = tuple([None] * len(dims)) - if self.shape is not None and self.dims is not None: - if len(self['dims']) != len(self['shape']): - raise ValueError("'dims' and 'shape' must be the same length") - - @property - def dtype(self): - ''' The data type of the attribute ''' - return self.get('dtype', None) - - @property - def value(self): - ''' The constant value of the attribute. "None" if this attribute is not constant ''' - return self.get('value', None) - - @property - def default_value(self): - ''' The default value of the attribute. "None" if this attribute has no default value ''' - return self.get('default_value', None) - - @property - def required(self): - ''' True if this attribute is required, False otherwise. ''' - return self.get('required', True) - - @property - def dims(self): - ''' The dimensions of this attribute's value ''' - return self.get('dims', None) - - @property - def shape(self): - ''' The shape of this attribute's value ''' - return self.get('shape', None) - - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this Spec class from a dictionary ''' - ret = super(AttributeSpec, cls).build_const_args(spec_dict) - if 'dtype' in ret: - if isinstance(ret['dtype'], dict): - ret['dtype'] = RefSpec.build_spec(ret['dtype']) - return ret - - -_attrbl_args = [ - {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - {'name': 'name', 'type': str, 'doc': 'the name of this base storage container', 'default': None}, - {'name': 'default_name', 'type': str, - 'doc': 'The default name of this base storage container', 'default': None}, - {'name': 'attributes', 'type': list, 'doc': 'the attributes on this group', 'default': list()}, - {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, - {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, - {'name': 'data_type_def', 'type': str, 'doc': 'the NWB type this specification represents', 'default': None}, - {'name': 'data_type_inc', 'type': (str, 'BaseStorageSpec'), - 'doc': 'the NWB type this specification extends', 'default': None}, -] - - -class BaseStorageSpec(Spec): - ''' A specification for any object that can hold attributes. ''' - - __inc_key = 'data_type_inc' - __def_key = 'data_type_def' - __type_key = 'data_type' - - @docval(*deepcopy(_attrbl_args)) # noqa: F821 - def __init__(self, **kwargs): - name, doc, parent, quantity, attributes, linkable, data_type_def, data_type_inc =\ - getargs('name', 'doc', 'parent', 'quantity', 'attributes', - 'linkable', 'data_type_def', 'data_type_inc', kwargs) - if name == NAME_WILDCARD and data_type_def is None and data_type_inc is None: - raise ValueError("Cannot create Group or Dataset spec with wildcard name \ - without specifying 'data_type_def' and/or 'data_type_inc'") - super(BaseStorageSpec, self).__init__(doc, name=name, parent=parent) - default_name = getargs('default_name', kwargs) - if default_name: - if name is not None: - warn("found 'default_name' with 'name' - ignoring 'default_name'") # noqa: F821 - else: - self['default_name'] = default_name - self.__attributes = dict() - if quantity in (ONE_OR_MANY, ZERO_OR_MANY): - if name != NAME_WILDCARD: - raise ValueError(("Cannot give specific name to something that can ", - "exist multiple times: name='%s', quantity='%s'" % (name, quantity))) - if quantity != DEF_QUANTITY: - self['quantity'] = quantity - if not linkable: - self['linkable'] = False - resolve = False - if data_type_inc is not None: - if isinstance(data_type_inc, BaseStorageSpec): - self[self.inc_key()] = data_type_inc.data_type_def - else: - self[self.inc_key()] = data_type_inc - if data_type_def is not None: - self.pop('required', None) - self[self.def_key()] = data_type_def - if data_type_inc is not None and isinstance(data_type_inc, BaseStorageSpec): - resolve = True - for attribute in attributes: - self.set_attribute(attribute) - self.__new_attributes = set(self.__attributes.keys()) - self.__overridden_attributes = set() - self.__resolved = False - if resolve: - self.resolve_spec(data_type_inc) - self.__resolved = True - - @property - def default_name(self): - '''The default name for this spec''' - return self.get('default_name', None) - - @property - def resolved(self): - return self.__resolved - - @property - def required(self): - ''' Whether or not the this spec represents a required field ''' - return self.quantity not in (ZERO_OR_ONE, ZERO_OR_MANY) - - @docval({'name': 'inc_spec', 'type': 'BaseStorageSpec', 'doc': 'the data type this specification represents'}) - def resolve_spec(self, **kwargs): - inc_spec = getargs('inc_spec', kwargs) - for attribute in inc_spec.attributes: - self.__new_attributes.discard(attribute) - if attribute.name in self.__attributes: - self.__overridden_attributes.add(attribute.name) - continue - self.set_attribute(attribute) - - @docval({'name': 'spec', 'type': (Spec, str), 'doc': 'the specification to check'}) - def is_inherited_spec(self, **kwargs): - ''' - Return True if this spec was inherited from the parent type, False otherwise - ''' - spec = getargs('spec', kwargs) - if isinstance(spec, Spec): - spec = spec.name - if spec in self.__attributes: - return self.is_inherited_attribute(spec) - return False - - @docval({'name': 'spec', 'type': (Spec, str), 'doc': 'the specification to check'}) - def is_overridden_spec(self, **kwargs): - ''' - Return True if this spec overrides a specification from the parent type, False otherwise - ''' - spec = getargs('spec', kwargs) - if isinstance(spec, Spec): - spec = spec.name - if spec in self.__attributes: - return self.is_overridden_attribute(spec) - return False - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to the Spec for'}) - def is_inherited_attribute(self, **kwargs): - ''' - Return True if the attribute was inherited from the parent type, False otherwise - ''' - name = getargs('name', kwargs) - if name not in self.__attributes: - raise ValueError("Attribute '%s' not found" % name) - return name not in self.__new_attributes - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to the Spec for'}) - def is_overridden_attribute(self, **kwargs): - ''' - Return True if the given attribute overrides the specification from the parent, False otherwise - ''' - name = getargs('name', kwargs) - if name not in self.__attributes: - raise ValueError("Attribute '%s' not found" % name) - return name not in self.__overridden_attributes - - def is_many(self): - return self.quantity not in (1, ZERO_OR_ONE) - - @classmethod - def get_data_type_spec(cls, data_type_def): - return AttributeSpec(cls.type_key(), 'the data type of this object', 'text', value=data_type_def) - - @classmethod - def get_namespace_spec(cls): - return AttributeSpec('namespace', 'the namespace for the data type of this object', 'text', required=False) - - @property - def attributes(self): - ''' The attributes for this specification ''' - return tuple(self.get('attributes', tuple())) - - @property - def linkable(self): - ''' True if object can be a link, False otherwise ''' - return self.get('linkable', True) - - @classmethod - def type_key(cls): - ''' Get the key used to store data type on an instance - - Override this method to use a different name for 'data_type' - ''' - return cls.__type_key - - @classmethod - def inc_key(cls): - ''' Get the key used to define a data_type include. - - Override this method to use a different keyword for 'data_type_inc' - ''' - return cls.__inc_key - - @classmethod - def def_key(cls): - ''' Get the key used to define a data_type definition. - - Override this method to use a different keyword for 'data_type_def' - ''' - return cls.__def_key - - @property - def data_type_inc(self): - ''' The data type of this specification ''' - return self.get(self.inc_key()) - - @property - def data_type_def(self): - ''' The data type this specification defines ''' - return self.get(self.def_key(), None) - - @property - def quantity(self): - ''' The number of times the object being specified should be present ''' - return self.get('quantity', DEF_QUANTITY) - - @docval(*deepcopy(_attr_args)) # noqa: F821 - def add_attribute(self, **kwargs): - ''' Add an attribute to this specification ''' - pargs, pkwargs = fmt_docval_args(AttributeSpec.__init__, kwargs) - spec = AttributeSpec(*pargs, **pkwargs) - self.set_attribute(spec) - return spec - - @docval({'name': 'spec', 'type': AttributeSpec, 'doc': 'the specification for the attribute to add'}) - def set_attribute(self, **kwargs): - ''' Set an attribute on this specification ''' - spec = kwargs.get('spec') - attributes = self.setdefault('attributes', list()) - if spec.parent is not None: - spec = AttributeSpec.build_spec(spec) - if spec.name in self.__attributes: - idx = -1 - for i, attribute in enumerate(attributes): - if attribute.name == spec.name: - idx = i - break - if idx >= 0: - attributes[idx] = spec - else: - raise ValueError('%s in __attributes but not in spec record' % spec.name) - else: - attributes.append(spec) - self.__attributes[spec.name] = spec - spec.parent = self - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to the Spec for'}) - def get_attribute(self, **kwargs): - ''' Get an attribute on this specification ''' - name = getargs('name', kwargs) - return self.__attributes.get(name) - - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this Spec class from a dictionary ''' - ret = super(BaseStorageSpec, cls).build_const_args(spec_dict) - if 'attributes' in ret: - ret['attributes'] = [AttributeSpec.build_spec(sub_spec) for sub_spec in ret['attributes']] - return ret - - -_dt_args = [ - {'name': 'name', 'type': str, 'doc': 'the name of this column'}, - {'name': 'doc', 'type': str, 'doc': 'a description about what this data type is'}, - {'name': 'dtype', 'type': (str, list, RefSpec), 'doc': 'the data type of this column'}, -] - - -class DtypeSpec(ConstructableDict): - '''A class for specifying a component of a compound type''' - - @docval(*_dt_args) - def __init__(self, **kwargs): - doc, name, dtype = getargs('doc', 'name', 'dtype', kwargs) - self['doc'] = doc - self['name'] = name - self.assertValidDtype(dtype) - self['dtype'] = dtype - - @property - def doc(self): - '''Documentation about this component''' - return self['doc'] - - @property - def name(self): - '''The name of this component''' - return self['name'] - - @property - def dtype(self): - ''' The data type of this component''' - return self['dtype'] - - @staticmethod - def assertValidDtype(dtype): - if isinstance(dtype, dict): - if _target_type_key not in dtype: - msg = "'dtype' must have the key '%s'" % _target_type_key - raise AssertionError(msg) - elif isinstance(dtype, RefSpec): - pass - else: - if dtype not in DtypeHelper.valid_primary_dtypes: - msg = "'dtype=%s' string not in valid primary data type: %s " % (str(dtype), - str(DtypeHelper.valid_primary_dtypes)) - raise AssertionError(msg) - return True - - @staticmethod - @docval({'name': 'spec', 'type': (str, dict), 'doc': 'the spec object to check'}, is_method=False) - def is_ref(**kwargs): - spec = getargs('spec', kwargs) - spec_is_ref = False - if isinstance(spec, dict): - if _target_type_key in spec: - spec_is_ref = True - elif 'dtype' in spec and isinstance(spec['dtype'], dict) and _target_type_key in spec['dtype']: - spec_is_ref = True - return spec_is_ref - - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this Spec class from a dictionary ''' - ret = super(DtypeSpec, cls).build_const_args(spec_dict) - if isinstance(ret['dtype'], list): - ret['dtype'] = list(map(cls.build_const_args, ret['dtype'])) - elif isinstance(ret['dtype'], dict): - ret['dtype'] = RefSpec.build_spec(ret['dtype']) - return ret - - -_dataset_args = [ - {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - {'name': 'dtype', 'type': (str, list, RefSpec), - 'doc': 'The data type of this attribute. Use a list of DtypeSpecs to specify a compound data type.', - 'default': None}, - {'name': 'name', 'type': str, 'doc': 'The name of this dataset', 'default': None}, - {'name': 'default_name', 'type': str, 'doc': 'The default name of this dataset', 'default': None}, - {'name': 'shape', 'type': (list, tuple), 'doc': 'the shape of this dataset', 'default': None}, - {'name': 'dims', 'type': (list, tuple), 'doc': 'the dimensions of this dataset', 'default': None}, - {'name': 'attributes', 'type': list, 'doc': 'the attributes on this group', 'default': list()}, - {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, - {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, - {'name': 'default_value', 'type': None, 'doc': 'a default value for this dataset', 'default': None}, - {'name': 'data_type_def', 'type': str, 'doc': 'the NWB type this specification represents', 'default': None}, - {'name': 'data_type_inc', 'type': (str, 'DatasetSpec'), - 'doc': 'the NWB type this specification extends', 'default': None}, -] - - -class DatasetSpec(BaseStorageSpec): - ''' Specification for datasets - - To specify a table-like dataset i.e. a compound data type. - ''' - - @docval(*deepcopy(_dataset_args)) # noqa: F821 - def __init__(self, **kwargs): - doc, shape, dims, dtype, default_value = popargs('doc', 'shape', 'dims', 'dtype', 'default_value', kwargs) - if shape is not None: - self['shape'] = shape - if dims is not None: - self['dims'] = dims - if 'shape' not in self: - self['shape'] = tuple([None] * len(dims)) - if self.shape is not None and self.dims is not None: - if len(self['dims']) != len(self['shape']): - raise ValueError("'dims' and 'shape' must be the same length") - if dtype is not None: - if isinstance(dtype, list): # Dtype is a compound data type - for _i, col in enumerate(dtype): - if not isinstance(col, DtypeSpec): - msg = 'must use DtypeSpec if defining compound dtype - found %s at element %d' % \ - (type(col), _i) - raise ValueError(msg) - self['dtype'] = dtype - elif isinstance(dtype, RefSpec): # Dtype is a reference - self['dtype'] = dtype - else: # Dtype is a string - self['dtype'] = dtype - if self['dtype'] not in DtypeHelper.valid_primary_dtypes: - raise ValueError('dtype %s not a valid primary data type %s' % - (self['dtype'], str(DtypeHelper.valid_primary_dtypes))) - super(DatasetSpec, self).__init__(doc, **kwargs) - if default_value is not None: - self['default_value'] = default_value - if self.name is not None: - valid_quant_vals = [1, 'zero_or_one', ZERO_OR_ONE] - if self.quantity not in valid_quant_vals: - raise ValueError("quantity %s invalid for spec with fixed name. Valid values are: %s" % - (self.quantity, str(valid_quant_vals))) - - @classmethod - def __get_prec_level(cls, dtype): - m = re.search('[0-9]+', dtype) - if m is not None: - prec = int(m.group()) - else: - prec = 32 - return (dtype[0], prec) - - @classmethod - def __is_sub_dtype(cls, orig, new): - if isinstance(orig, RefSpec): - if not isinstance(new, RefSpec): - return False - return orig == new - else: - orig_prec = cls.__get_prec_level(orig) - new_prec = cls.__get_prec_level(new) - if orig_prec[0] != new_prec[0]: - # cannot extend int to float and vice-versa - return False - return new_prec >= orig_prec - - @docval({'name': 'inc_spec', 'type': 'DatasetSpec', 'doc': 'the data type this specification represents'}) - def resolve_spec(self, **kwargs): - inc_spec = getargs('inc_spec', kwargs) - if isinstance(self.dtype, list): - # merge the new types - inc_dtype = inc_spec.dtype - if isinstance(inc_dtype, str): - msg = 'Cannot extend simple data type to compound data type' - raise ValueError(msg) - order = OrderedDict() - if inc_dtype is not None: - for dt in inc_dtype: - order[dt['name']] = dt - for dt in self.dtype: - name = dt['name'] - if name in order: - # verify that the exension has supplied - # a valid subtyping of existing type - orig = order[name].dtype - new = dt.dtype - if not self.__is_sub_dtype(orig, new): - msg = 'Cannot extend %s to %s' % (str(orig), str(new)) - raise ValueError(msg) - order[name] = dt - self['dtype'] = list(order.values()) - super(DatasetSpec, self).resolve_spec(inc_spec) - - @property - def dims(self): - ''' The dimensions of this Dataset ''' - return self.get('dims', None) - - @property - def dtype(self): - ''' The data type of the Dataset ''' - return self.get('dtype', None) - - @property - def shape(self): - ''' The shape of the dataset ''' - return self.get('shape', None) - - @property - def default_value(self): - '''The default value of the dataset or None if not specified''' - return self.get('default_value', None) - - @classmethod - def __check_dim(cls, dim, data): - return True - - @classmethod - def dtype_spec_cls(cls): - ''' The class to use when constructing DtypeSpec objects - - Override this if extending to use a class other than DtypeSpec to build - dataset specifications - ''' - return DtypeSpec - - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this Spec class from a dictionary ''' - ret = super(DatasetSpec, cls).build_const_args(spec_dict) - if 'dtype' in ret: - if isinstance(ret['dtype'], list): - ret['dtype'] = list(map(cls.dtype_spec_cls().build_spec, ret['dtype'])) - elif isinstance(ret['dtype'], dict): - ret['dtype'] = RefSpec.build_spec(ret['dtype']) - return ret - - -_link_args = [ - {'name': 'doc', 'type': str, 'doc': 'a description about what this link represents'}, - {'name': _target_type_key, 'type': str, 'doc': 'the target type GroupSpec or DatasetSpec'}, - {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, - {'name': 'name', 'type': str, 'doc': 'the name of this link', 'default': None} -] - - -class LinkSpec(Spec): - - @docval(*_link_args) - def __init__(self, **kwargs): - doc, target_type, name, quantity = popargs('doc', _target_type_key, 'name', 'quantity', kwargs) - super(LinkSpec, self).__init__(doc, name, **kwargs) - self[_target_type_key] = target_type - if quantity != 1: - self['quantity'] = quantity - - @property - def target_type(self): - ''' The data type of target specification ''' - return self.get(_target_type_key) - - @property - def data_type_inc(self): - ''' The data type of target specification ''' - return self.get(_target_type_key) - - def is_many(self): - return self.quantity not in (1, ZERO_OR_ONE) - - @property - def quantity(self): - ''' The number of times the object being specified should be present ''' - return self.get('quantity', DEF_QUANTITY) - - @property - def required(self): - ''' Whether or not the this spec represents a required field ''' - return self.quantity not in (ZERO_OR_ONE, ZERO_OR_MANY) - - -_group_args = [ - {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - {'name': 'name', 'type': str, 'doc': 'the name of this group', 'default': None}, - {'name': 'default_name', 'type': str, 'doc': 'The default name of this group', 'default': None}, - {'name': 'groups', 'type': list, 'doc': 'the subgroups in this group', 'default': list()}, - {'name': 'datasets', 'type': list, 'doc': 'the datasets in this group', 'default': list()}, - {'name': 'attributes', 'type': list, 'doc': 'the attributes on this group', 'default': list()}, - {'name': 'links', 'type': list, 'doc': 'the links in this group', 'default': list()}, - {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, - {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, - {'name': 'data_type_def', 'type': str, 'doc': 'the NWB type this specification represents', 'default': None}, - {'name': 'data_type_inc', 'type': (str, 'GroupSpec'), - 'doc': 'the NWB type this specification data_type_inc', 'default': None}, -] - - -class GroupSpec(BaseStorageSpec): - ''' Specification for groups - ''' - - @docval(*deepcopy(_group_args)) # noqa: F821 - def __init__(self, **kwargs): - doc, groups, datasets, links = popargs('doc', 'groups', 'datasets', 'links', kwargs) - self.__data_types = dict() - self.__groups = dict() - for group in groups: - self.set_group(group) - self.__datasets = dict() - for dataset in datasets: - self.set_dataset(dataset) - self.__links = dict() - for link in links: - self.set_link(link) - self.__new_data_types = set(self.__data_types.keys()) - self.__new_datasets = set(self.__datasets.keys()) - self.__overridden_datasets = set() - self.__new_links = set(self.__links.keys()) - self.__overridden_links = set() - self.__new_groups = set(self.__groups.keys()) - self.__overridden_groups = set() - super(GroupSpec, self).__init__(doc, **kwargs) - - @docval({'name': 'inc_spec', 'type': 'GroupSpec', 'doc': 'the data type this specification represents'}) - def resolve_spec(self, **kwargs): - inc_spec = getargs('inc_spec', kwargs) - data_types = list() - # resolve inherited datasets - for dataset in inc_spec.datasets: - # if not (dataset.data_type_def is None and dataset.data_type_inc is None): - if dataset.name is None: - data_types.append(dataset) - continue - self.__new_datasets.discard(dataset.name) - if dataset.name in self.__datasets: - self.__datasets[dataset.name].resolve_spec(dataset) - self.__overridden_datasets.add(dataset.name) - else: - self.set_dataset(dataset) - # resolve inherited groups - for group in inc_spec.groups: - # if not (group.data_type_def is None and group.data_type_inc is None): - if group.name is None: - data_types.append(group) - continue - self.__new_groups.discard(group.name) - if group.name in self.__groups: - self.__groups[group.name].resolve_spec(group) - self.__overridden_groups.add(group.name) - else: - self.set_group(group) - # resolve inherited links - for link in inc_spec.links: - if link.name is None: - data_types.append(link) - self.__new_links.discard(link.name) - if link.name in self.__links: - self.__overridden_links.add(link.name) - continue - self.set_link(link) - # resolve inherited data_types - for dt_spec in data_types: - if isinstance(dt_spec, LinkSpec): - dt = dt_spec.target_type - else: - dt = dt_spec.data_type_def - if dt is None: - dt = dt_spec.data_type_inc - self.__new_data_types.discard(dt) - existing_dt_spec = self.get_data_type(dt) - if existing_dt_spec is None or \ - ((isinstance(existing_dt_spec, list) or existing_dt_spec.name is not None)) and \ - dt_spec.name is None: - if isinstance(dt_spec, DatasetSpec): - self.set_dataset(dt_spec) - elif isinstance(dt_spec, GroupSpec): - self.set_group(dt_spec) - else: - self.set_link(dt_spec) - super(GroupSpec, self).resolve_spec(inc_spec) - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the dataset'}, - raises="ValueError, if 'name' is not part of this spec") - def is_inherited_dataset(self, **kwargs): - '''Return true if a dataset with the given name was inherited''' - name = getargs('name', kwargs) - if name not in self.__datasets: - raise ValueError("Dataset '%s' not found in spec" % name) - return name not in self.__new_datasets - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the dataset'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_dataset(self, **kwargs): - '''Return true if a dataset with the given name overrides a specification from the parent type''' - name = getargs('name', kwargs) - if name not in self.__datasets: - raise ValueError("Dataset '%s' not found in spec" % name) - return name in self.__overridden_datasets - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'}, - raises="ValueError, if 'name' is not part of this spec") - def is_inherited_group(self, **kwargs): - '''Return true if a group with the given name was inherited''' - name = getargs('name', kwargs) - if name not in self.__groups: - raise ValueError("Group '%s' not found in spec" % name) - return name not in self.__new_groups - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_group(self, **kwargs): - '''Return true if a group with the given name overrides a specification from the parent type''' - name = getargs('name', kwargs) - if name not in self.__groups: - raise ValueError("Group '%s' not found in spec" % name) - return name in self.__overridden_groups - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the link'}, - raises="ValueError, if 'name' is not part of this spec") - def is_inherited_link(self, **kwargs): - '''Return true if a link with the given name was inherited''' - name = getargs('name', kwargs) - if name not in self.__links: - raise ValueError("Link '%s' not found in spec" % name) - return name not in self.__new_links - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the link'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_link(self, **kwargs): - '''Return true if a link with the given name overrides a specification from the parent type''' - name = getargs('name', kwargs) - if name not in self.__links: - raise ValueError("Link '%s' not found in spec" % name) - return name in self.__overridden_links - - @docval({'name': 'spec', 'type': (Spec, str), 'doc': 'the specification to check'}) - def is_inherited_spec(self, **kwargs): - ''' Returns 'True' if specification was inherited from a parent type ''' - spec = getargs('spec', kwargs) - if isinstance(spec, Spec): - name = spec.name - if name is None: - if isinstance(spec, LinkSpec): - name = spec.target_type - else: - name = spec.data_type_def - if name is None: - name = spec.data_type_inc - if name is None: - raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') - spec = name - if spec in self.__links: - return self.is_inherited_link(spec) - elif spec in self.__groups: - return self.is_inherited_group(spec) - elif spec in self.__datasets: - return self.is_inherited_dataset(spec) - elif spec in self.__data_types: - return self.is_inherited_type(spec) - else: - if super(GroupSpec, self).is_inherited_spec(spec): - return True - else: - for s in self.__datasets: - if self.is_inherited_dataset(s): - if self.__datasets[s].get_attribute(spec) is not None: - return True - for s in self.__groups: - if self.is_inherited_group(s): - if self.__groups[s].get_attribute(spec) is not None: - return True - return False - - @docval({'name': 'spec', 'type': (Spec, str), 'doc': 'the specification to check'}) - def is_overridden_spec(self, **kwargs): - ''' Returns 'True' if specification was inherited from a parent type ''' - spec = getargs('spec', kwargs) - if isinstance(spec, Spec): - name = spec.name - if name is None: - if spec.is_many(): # this is a wildcard spec, so it cannot be overridden - return False - name = spec.data_type_def - if name is None: - name = spec.data_type_inc - if name is None: - raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') - spec = name - if spec in self.__links: - return self.is_overridden_link(spec) - elif spec in self.__groups: - return self.is_overridden_group(spec) - elif spec in self.__datasets: - return self.is_overridden_dataset(spec) - elif spec in self.__data_types: - return self.is_overridden_type(spec) - else: - if super(GroupSpec, self).is_overridden_spec(spec): # check if overridden attribute - return True - else: - for s in self.__datasets: - if self.is_overridden_dataset(s): - if self.__datasets[s].is_overridden_spec(spec): - return True - for s in self.__groups: - if self.is_overridden_group(s): - if self.__groups[s].is_overridden_spec(spec): - return True - return False - - @docval({'name': 'spec', 'type': (BaseStorageSpec, str), 'doc': 'the specification to check'}) - def is_inherited_type(self, **kwargs): - ''' Returns True if `spec` represents a spec that was inherited from an included data_type ''' - spec = getargs('spec', kwargs) - if isinstance(spec, BaseStorageSpec): - if spec.data_type_def is None: - raise ValueError('cannot check if something was inherited if it does not have a %s' % self.def_key()) - spec = spec.data_type_def - return spec not in self.__new_data_types - - @docval({'name': 'spec', 'type': (BaseStorageSpec, str), 'doc': 'the specification to check'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_type(self, **kwargs): - ''' Returns True if `spec` represents a spec that was overriden by the subtype''' - spec = getargs('spec', kwargs) - if isinstance(spec, BaseStorageSpec): - if spec.data_type_def is None: - raise ValueError('cannot check if something was inherited if it does not have a %s' % self.def_key()) - spec = spec.data_type_def - return spec not in self.__new_data_types - - def __add_data_type_inc(self, spec): - dt = None - if hasattr(spec, 'data_type_def') and spec.data_type_def is not None: - dt = spec.data_type_def - elif hasattr(spec, 'data_type_inc') and spec.data_type_inc is not None: - dt = spec.data_type_inc - if not dt: - raise TypeError("spec does not have '%s' or '%s' defined" % (self.def_key(), self.inc_key())) - if dt in self.__data_types: - curr = self.__data_types[dt] - if curr is spec: - return - if spec.name is None: - if isinstance(curr, list): - self.__data_types[dt] = spec - else: - if curr.name is None: - raise TypeError('Cannot have multiple data types of the same type without specifying name') - else: - # unnamed data types will be stored as data_types - self.__data_types[dt] = spec - else: - if isinstance(curr, list): - self.__data_types[dt].append(spec) - else: - if curr.name is None: - # leave the existing data type as is, since the new one can be retrieved by name - return - else: - # store both specific instances of a data type - self.__data_types[dt] = [curr, spec] - else: - self.__data_types[dt] = spec - - @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to retrieve'}) - def get_data_type(self, **kwargs): - ''' - Get a specification by "data_type" - ''' - ndt = getargs('data_type', kwargs) - return self.__data_types.get(ndt, None) - - @property - def groups(self): - ''' The groups specificed in this GroupSpec ''' - return tuple(self.get('groups', tuple())) - - @property - def datasets(self): - ''' The datasets specificed in this GroupSpec ''' - return tuple(self.get('datasets', tuple())) - - @property - def links(self): - ''' The links specificed in this GroupSpec ''' - return tuple(self.get('links', tuple())) - - @docval(*deepcopy(_group_args)) # noqa: F821 - def add_group(self, **kwargs): - ''' Add a new specification for a subgroup to this group specification ''' - doc = kwargs.pop('doc') - spec = self.__class__(doc, **kwargs) - self.set_group(spec) - return spec - - @docval({'name': 'spec', 'type': ('GroupSpec'), 'doc': 'the specification for the subgroup'}) - def set_group(self, **kwargs): - ''' Add the given specification for a subgroup to this group specification ''' - spec = getargs('spec', kwargs) - if spec.parent is not None: - spec = self.build_spec(spec) - if spec.name == NAME_WILDCARD: - if spec.data_type_inc is not None or spec.data_type_def is not None: - self.__add_data_type_inc(spec) - else: - raise TypeError("must specify 'name' or 'data_type_inc' in Group spec") - else: - if spec.data_type_inc is not None or spec.data_type_def is not None: - self.__add_data_type_inc(spec) - self.__groups[spec.name] = spec - self.setdefault('groups', list()).append(spec) - spec.parent = self - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the group to the Spec for'}) - def get_group(self, **kwargs): - ''' Get a specification for a subgroup to this group specification ''' - name = getargs('name', kwargs) - return self.__groups.get(name, self.__links.get(name)) - - @docval(*deepcopy(_dataset_args)) # noqa: F821 - def add_dataset(self, **kwargs): - ''' Add a new specification for a dataset to this group specification ''' - doc = kwargs.pop('doc') - spec = self.dataset_spec_cls()(doc, **kwargs) - self.set_dataset(spec) - return spec - - @docval({'name': 'spec', 'type': 'DatasetSpec', 'doc': 'the specification for the dataset'}) - def set_dataset(self, **kwargs): - ''' Add the given specification for a dataset to this group specification ''' - spec = getargs('spec', kwargs) - if spec.parent is not None: - spec = self.dataset_spec_cls().build_spec(spec) - if spec.name == NAME_WILDCARD: - if spec.data_type_inc is not None or spec.data_type_def is not None: - self.__add_data_type_inc(spec) - else: - raise TypeError("must specify 'name' or 'data_type_inc' in Dataset spec") - else: - if spec.data_type_inc is not None or spec.data_type_def is not None: - self.__add_data_type_inc(spec) - self.__datasets[spec.name] = spec - self.setdefault('datasets', list()).append(spec) - spec.parent = self - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the dataset to the Spec for'}) - def get_dataset(self, **kwargs): - ''' Get a specification for a dataset to this group specification ''' - name = getargs('name', kwargs) - return self.__datasets.get(name, self.__links.get(name)) - - @docval(*_link_args) - def add_link(self, **kwargs): - ''' Add a new specification for a link to this group specification ''' - doc, target_type = popargs('doc', _target_type_key, kwargs) - spec = self.link_spec_cls()(doc, target_type, **kwargs) - self.set_link(spec) - return spec - - @docval({'name': 'spec', 'type': 'LinkSpec', 'doc': 'the specification for the object to link to'}) - def set_link(self, **kwargs): - ''' Add a given specification for a link to this group specification ''' - spec = getargs('spec', kwargs) - if spec.parent is not None: - spec = self.link_spec_cls().build_spec(spec) - if spec.name == NAME_WILDCARD: - if spec.data_type_inc is not None or spec.data_type_def is not None: - self.__add_data_type_inc(spec) - else: - raise TypeError("must specify 'name' or 'data_type_inc' in Dataset spec") - else: - self.__links[spec.name] = spec - self.setdefault('links', list()).append(spec) - spec.parent = self - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the link to the Spec for'}) - def get_link(self, **kwargs): - ''' Get a specification for a link to this group specification ''' - name = getargs('name', kwargs) - return self.__links.get(name) - - @classmethod - def dataset_spec_cls(cls): - ''' The class to use when constructing DatasetSpec objects - - Override this if extending to use a class other than DatasetSpec to build - dataset specifications - ''' - return DatasetSpec - - @classmethod - def link_spec_cls(cls): - ''' The class to use when constructing LinkSpec objects - - Override this if extending to use a class other than LinkSpec to build - link specifications - ''' - return LinkSpec - - @classmethod - def build_const_args(cls, spec_dict): - ''' Build constructor arguments for this Spec class from a dictionary ''' - ret = super(GroupSpec, cls).build_const_args(spec_dict) - if 'datasets' in ret: - ret['datasets'] = list(map(cls.dataset_spec_cls().build_spec, ret['datasets'])) - if 'groups' in ret: - ret['groups'] = list(map(cls.build_spec, ret['groups'])) - if 'links' in ret: - ret['links'] = list(map(cls.link_spec_cls().build_spec, ret['links'])) - return ret diff --git a/src/pynwb/form/spec/write.py b/src/pynwb/form/spec/write.py deleted file mode 100644 index 3051007c4..000000000 --- a/src/pynwb/form/spec/write.py +++ /dev/null @@ -1,191 +0,0 @@ -import copy -import json -import ruamel.yaml as yaml -import os.path -from collections import OrderedDict -from six import with_metaclass -from abc import ABCMeta, abstractmethod - -from .namespace import SpecNamespace -from .spec import GroupSpec, DatasetSpec -from .catalog import SpecCatalog - -from ..utils import docval, getargs, popargs - - -class SpecWriter(with_metaclass(ABCMeta, object)): - - @abstractmethod - def write_spec(self, spec_file_dict, path): - pass - - @abstractmethod - def write_namespace(self, namespace, path): - pass - - -class YAMLSpecWriter(SpecWriter): - - @docval({'name': 'outdir', - 'type': str, - 'doc': 'the path to write the directory to output the namespace and specs too', 'default': '.'}) - def __init__(self, **kwargs): - self.__outdir = getargs('outdir', kwargs) - - def __dump_spec(self, specs, stream): - specs_plain_dict = json.loads(json.dumps(specs)) - yaml.main.safe_dump(specs_plain_dict, stream, default_flow_style=False) - - def write_spec(self, spec_file_dict, path): - out_fullpath = os.path.join(self.__outdir, path) - spec_plain_dict = json.loads(json.dumps(spec_file_dict)) - sorted_data = self.sort_keys(spec_plain_dict) - with open(out_fullpath, 'w') as fd_write: - yaml.dump(sorted_data, fd_write, Dumper=yaml.dumper.RoundTripDumper) - - def write_namespace(self, namespace, path): - with open(os.path.join(self.__outdir, path), 'w') as stream: - self.__dump_spec({'namespaces': [namespace]}, stream) - - def reorder_yaml(self, path): - """ - Open a YAML file, load it as python data, sort the data, and write it back out to the - same path. - """ - with open(path, 'rb') as fd_read: - data = yaml.load(fd_read, Loader=yaml.loader.RoundTripLoader) - self.write_spec(data, path) - - def sort_keys(self, obj): - - # Represent None as null - def my_represent_none(self, data): - return self.represent_scalar(u'tag:yaml.org,2002:null', u'null') - yaml.representer.RoundTripRepresenter.add_representer(type(None), my_represent_none) - - order = ['neurodata_type_def', 'neurodata_type_inc', 'name', 'dtype', 'doc', - 'attributes', 'datasets', 'groups'] - if isinstance(obj, dict): - keys = list(obj.keys()) - for k in order[::-1]: - if k in keys: - keys.remove(k) - keys.insert(0, k) - return yaml.comments.CommentedMap( - yaml.compat.ordereddict([(k, self.sort_keys(obj[k])) for k in keys]) - ) - elif isinstance(obj, list): - return [self.sort_keys(v) for v in obj] - elif isinstance(obj, tuple): - return (self.sort_keys(v) for v in obj) - else: - return obj - - -class NamespaceBuilder(object): - ''' A class for building namespace and spec files ''' - - @docval({'name': 'doc', 'type': str, 'doc': 'a description about what name namespace represents'}, - {'name': 'name', 'type': str, 'doc': 'the name of namespace'}, - {'name': 'full_name', 'type': str, 'doc': 'extended full name of name namespace', 'default': None}, - {'name': 'version', 'type': (str, tuple, list), 'doc': 'Version number of the namespace', 'default': None}, - {'name': 'author', 'type': (str, list), 'doc': 'Author or list of authors.', 'default': None}, - {'name': 'contact', 'type': (str, list), - 'doc': 'List of emails. Ordering should be the same as for author', 'default': None}, - {'name': 'namespace_cls', 'type': type, 'doc': 'the SpecNamespace type', 'default': SpecNamespace}) - def __init__(self, **kwargs): - ns_cls = popargs('namespace_cls', kwargs) - self.__ns_args = copy.deepcopy(kwargs) - self.__namespaces = OrderedDict() - self.__sources = OrderedDict() - self.__catalog = SpecCatalog() - self.__dt_key = ns_cls.types_key() - - @docval({'name': 'source', 'type': str, 'doc': 'the path to write the spec to'}, - {'name': 'spec', 'type': (GroupSpec, DatasetSpec), 'doc': 'the Spec to add'}) - def add_spec(self, **kwargs): - ''' Add a Spec to the namespace ''' - source, spec = getargs('source', 'spec', kwargs) - self.__catalog.auto_register(spec, source) - self.add_source(source) - self.__sources[source].setdefault(self.__dt_key, list()).append(spec) - - @docval({'name': 'source', 'type': str, 'doc': 'the path to write the spec to'}) - def add_source(self, **kwargs): - ''' Add a source file to the namespace ''' - source = getargs('source', kwargs) - if '/' in source or source[0] == '.': - raise ValueError('source must be a base file') - self.__sources.setdefault(source, {'source': source}) - - @docval({'name': 'data_type', 'type': str, 'doc': 'the data type to include'}, - {'name': 'source', 'type': str, 'doc': 'the source file to include the type from', 'default': None}, - {'name': 'namespace', 'type': str, - 'doc': 'the namespace from which to include the data type', 'default': None}) - def include_type(self, **kwargs): - ''' Include a data type from an existing namespace or source ''' - dt, src, ns = getargs('data_type', 'source', 'namespace', kwargs) - if src is not None: - self.add_source(source) # noqa: F821 - self.__sources[path].setdefault(self.__dt_key, list()).append(dt) # noqa: F821 - elif ns is not None: - self.include_namespace(ns) - self.__namespaces[ns].setdefault(self.__dt_key, list()).append(dt) - else: - raise ValueError("must specify 'source' or 'namespace' when including type") - - @docval({'name': 'namespace', 'type': str, 'doc': 'the namespace to include'}) - def include_namespace(self, **kwargs): - ''' Include an entire namespace ''' - namespace = getargs('namespace', kwargs) - self.__namespaces.setdefault(namespace, {'namespace': namespace}) - - @docval({'name': 'path', 'type': str, 'doc': 'the path to write the spec to'}, - {'name': 'outdir', - 'type': str, - 'doc': 'the path to write the directory to output the namespace and specs too', 'default': '.'}, - {'name': 'writer', - 'type': SpecWriter, - 'doc': 'the SpecWriter to use to write the namespace', 'default': None}) - def export(self, **kwargs): - ''' Export the namespace to the given path. - - All new specification source files will be written in the same directory as the - given path. - ''' - ns_path, writer = getargs('path', 'writer', kwargs) - if writer is None: - writer = YAMLSpecWriter(outdir=getargs('outdir', kwargs)) - ns_args = copy.copy(self.__ns_args) - ns_args['schema'] = list() - for ns, info in self.__namespaces.items(): - ns_args['schema'].append(info) - for path, info in self.__sources.items(): - out = SpecFileBuilder() - dts = list() - for spec in info[self.__dt_key]: - if isinstance(spec, str): - dts.append(spec) - else: - out.add_spec(spec) - item = {'source': path} - if out and dts: - raise ValueError('cannot include from source if writing to source') - elif dts: - item[self.__dt_key] = dts - elif out: - writer.write_spec(out, path) - ns_args['schema'].append(item) - namespace = SpecNamespace.build_namespace(**ns_args) - writer.write_namespace(namespace, ns_path) - - -class SpecFileBuilder(dict): - - @docval({'name': 'spec', 'type': (GroupSpec, DatasetSpec), 'doc': 'the Spec to add'}) - def add_spec(self, **kwargs): - spec = getargs('spec', kwargs) - if isinstance(spec, GroupSpec): - self.setdefault('groups', list()).append(spec) - elif isinstance(spec, DatasetSpec): - self.setdefault('datasets', list()).append(spec) diff --git a/src/pynwb/form/utils.py b/src/pynwb/form/utils.py deleted file mode 100644 index d1f9855ff..000000000 --- a/src/pynwb/form/utils.py +++ /dev/null @@ -1,583 +0,0 @@ -import copy as _copy -import itertools as _itertools -from abc import ABCMeta - -import h5py -import numpy as np -import six -from six import raise_from, text_type, binary_type - - -__macros = { - 'array_data': [np.ndarray, list, tuple, h5py.Dataset], - 'scalar_data': [str, int, float], -} - - -def docval_macro(macro): - def _dec(cls): - if macro not in __macros: - __macros[macro] = list() - __macros[macro].append(cls) - return cls - return _dec - - -def __type_okay(value, argtype, allow_none=False): - """Check a value against a type - - The difference between this function and :py:func:`isinstance` is that - it allows specifying a type as a string. Furthermore, strings allow for specifying more general - types, such as a simple numeric type (i.e. ``argtype``="num"). - - Args: - value (any): the value to check - argtype (type, str): the type to check for - allow_none (bool): whether or not to allow None as a valid value - - - Returns: - bool: True if value is a valid instance of argtype - """ - if value is None: - return allow_none - if isinstance(argtype, str): - if argtype in __macros: - return __type_okay(value, __macros[argtype], allow_none=allow_none) - elif argtype == 'int': - return __is_int(value) - elif argtype == 'float': - return __is_float(value) - return argtype in [cls.__name__ for cls in value.__class__.__mro__] - elif isinstance(argtype, type): - if argtype == six.text_type: - return isinstance(value, six.text_type) or isinstance(value, six.string_types) - elif argtype == str: - return isinstance(value, six.string_types) - elif argtype is int: - return __is_int(value) - elif argtype is float: - return __is_float(value) - return isinstance(value, argtype) - elif isinstance(argtype, tuple) or isinstance(argtype, list): - return any(__type_okay(value, i) for i in argtype) - else: # argtype is None - return True - - -def __shape_okay_multi(value, argshape): - if type(argshape[0]) in (tuple, list): # if multiple shapes are present - return any(__shape_okay(value, a) for a in argshape) - else: - return __shape_okay(value, argshape) - - -def __shape_okay(value, argshape): - valshape = get_data_shape(value) - if not len(valshape) == len(argshape): - return False - for a, b in zip(valshape, argshape): - if b not in (a, None): - return False - return True - - -def __is_int(value): - return any(isinstance(value, i) for i in (int, np.int8, np.int16, np.int32, np.int64)) - - -def __is_float(value): - SUPPORTED_FLOAT_TYPES = [float, np.float16, np.float32, np.float64] - if hasattr(np, "float128"): - SUPPORTED_FLOAT_TYPES.append(np.float128) - return any(isinstance(value, i) for i in SUPPORTED_FLOAT_TYPES) - - -def __format_type(argtype): - if isinstance(argtype, str): - return argtype - elif isinstance(argtype, type): - return argtype.__name__ - elif isinstance(argtype, tuple) or isinstance(argtype, list): - types = [__format_type(i) for i in argtype] - if len(types) > 1: - return "%s or %s" % (", ".join(types[:-1]), types[-1]) - else: - return types[0] - elif argtype is None: - return "NoneType" - else: - raise ValueError("argtype must be a type, str, list, or tuple") - - -def __parse_args(validator, args, kwargs, enforce_type=True, enforce_shape=True, allow_extra=False): # noqa: 901 - """ - Internal helper function used by the docval decorator to parse and validate function arguments - - :param validator: List of dicts from docval with the description of the arguments - :param args: List of the values of positional arguments supplied by the caller - :param kwargs: Dict keyword arguments supplied by the caller where keys are the argument name and - values are the argument value. - :param enforce_type: Boolean indicating whether the type of arguments should be enforced - :param enforce_shape: Boolean indicating whether the dimensions of array arguments - should be enforced if possible. - - :return: Dict with: - * 'args' : Dict all arguments where keys are the names and values are the values of the arguments. - * 'errors' : List of string with error messages - """ - ret = dict() - type_errors = list() - value_errors = list() - argsi = 0 - extras = dict(kwargs) - try: - it = iter(validator) - arg = next(it) - # catch unsupported keys - allowable_terms = ('name', 'doc', 'type', 'shape', 'default', 'help') - unsupported_terms = set(arg.keys()) - set(allowable_terms) - if unsupported_terms: - raise ValueError('docval for {}: {} are not supported by docval'.format(arg['name'], - list(unsupported_terms))) - # process positional arguments - while True: - # - if 'default' in arg: - break - argname = arg['name'] - argval_set = False - if argname in kwargs: - argval = kwargs.get(argname) - extras.pop(argname, None) - argval_set = True - elif argsi < len(args): - argval = args[argsi] - argval_set = True - - if not argval_set: - type_errors.append("missing argument '%s'" % argname) - else: - if argname in ret: - type_errors.append("'got multiple arguments for '%s" % argname) - else: - if enforce_type: - if not __type_okay(argval, arg['type']): - fmt_val = (argname, type(argval).__name__, __format_type(arg['type'])) - type_errors.append("incorrect type for '%s' (got '%s', expected '%s')" % fmt_val) - if enforce_shape and 'shape' in arg: - if not __shape_okay_multi(argval, arg['shape']): - fmt_val = (argname, get_data_shape(argval), arg['shape']) - value_errors.append("incorrect shape for '%s' (got '%s, expected '%s')" % fmt_val) - ret[argname] = argval - argsi += 1 - arg = next(it) - while True: - argname = arg['name'] - if argname in kwargs: - ret[argname] = kwargs.get(argname) - extras.pop(argname, None) - elif len(args) > argsi: - ret[argname] = args[argsi] - argsi += 1 - else: - ret[argname] = arg['default'] - argval = ret[argname] - if enforce_type: - if not __type_okay(argval, arg['type'], arg['default'] is None): - fmt_val = (argname, type(argval).__name__, __format_type(arg['type'])) - type_errors.append("incorrect type for '%s' (got '%s', expected '%s')" % fmt_val) - if enforce_shape and 'shape' in arg and argval is not None: - if not __shape_okay_multi(argval, arg['shape']): - fmt_val = (argname, get_data_shape(argval), arg['shape']) - value_errors.append("incorrect shape for '%s' (got '%s, expected '%s')" % fmt_val) - arg = next(it) - except StopIteration: - pass - if not allow_extra: - for key in extras.keys(): - type_errors.append("unrecognized argument: '%s'" % key) - else: - # TODO: Extras get stripped out if function arguments are composed with fmt_docval_args. - # allow_extra needs to be tracked on a function so that fmt_docval_args doesn't strip them out - for key in extras.keys(): - ret[key] = extras[key] - return {'args': ret, 'type_errors': type_errors, 'value_errors': value_errors} - - -def __sort_args(validator): - pos = list() - kw = list() - for arg in validator: - if "default" in arg: - kw.append(arg) - else: - pos.append(arg) - return list(_itertools.chain(pos, kw)) - - -docval_attr_name = '__docval__' -__docval_args_loc = 'args' - - -# TODO: write unit tests for get_docval* functions -def get_docval(func): - '''get_docval(func) - Get a copy of docval arguments for a function - ''' - func_docval = getattr(func, docval_attr_name, None) - if func_docval: - return tuple(func_docval[__docval_args_loc]) - else: - return tuple() - -# def docval_wrap(func, is_method=True): -# if is_method: -# @docval(*get_docval(func)) -# def method(self, **kwargs): -# -# return call_docval_args(func, kwargs) -# return method -# else: -# @docval(*get_docval(func)) -# def static_method(**kwargs): -# return call_docval_args(func, kwargs) -# return method - - -def fmt_docval_args(func, kwargs): - ''' Separate positional and keyword arguments - - Useful for methods that wrap other methods - ''' - func_docval = getattr(func, docval_attr_name, None) - ret_args = list() - ret_kwargs = dict() - kwargs_copy = _copy.copy(kwargs) - if func_docval: - for arg in func_docval[__docval_args_loc]: - val = kwargs_copy.pop(arg['name'], None) - if 'default' in arg: - if val is not None: - ret_kwargs[arg['name']] = val - else: - ret_args.append(val) - if func_docval['allow_extra']: - ret_kwargs.update(kwargs_copy) - else: - raise ValueError('no docval found on %s' % str(func)) - return ret_args, ret_kwargs - - -def call_docval_func(func, kwargs): - fargs, fkwargs = fmt_docval_args(func, kwargs) - return func(*fargs, **fkwargs) - - -def __resolve_type(t): - if t is None: - return t - if isinstance(t, str): - if t in __macros: - return tuple(__macros[t]) - else: - return t - elif isinstance(t, type): - return t - elif isinstance(t, (list, tuple)): - ret = list() - for i in t: - resolved = __resolve_type(i) - if isinstance(resolved, tuple): - ret.extend(resolved) - else: - ret.append(resolved) - return tuple(ret) - else: - msg = "argtype must be a type, a str, a list, a tuple, or None - got %s" % type(t) - raise ValueError(msg) - - -def docval(*validator, **options): - '''A decorator for documenting and enforcing type for instance method arguments. - - This decorator takes a list of dictionaries that specify the method parameters. These - dictionaries are used for enforcing type and building a Sphinx docstring. - - The first arguments are dictionaries that specify the positional - arguments and keyword arguments of the decorated function. These dictionaries - must contain the following keys: ``'name'``, ``'type'``, and ``'doc'``. This will define a - positional argument. To define a keyword argument, specify a default value - using the key ``'default'``. To validate the dimensions of an input array - add the optional ``'shape'`` parameter. - - The decorated method must take ``self`` and ``**kwargs`` as arguments. - - When using this decorator, the functions :py:func:`getargs` and - :py:func:`popargs` can be used for easily extracting arguments from - kwargs. - - The following code example demonstrates the use of this decorator: - - .. code-block:: python - - @docval({'name': 'arg1':, 'type': str, 'doc': 'this is the first positional argument'}, - {'name': 'arg2':, 'type': int, 'doc': 'this is the second positional argument'}, - {'name': 'kwarg1':, 'type': (list, tuple), 'doc': 'this is a keyword argument', 'default': list()}, - returns='foo object', rtype='Foo')) - def foo(self, **kwargs): - arg1, arg2, kwarg1 = getargs('arg1', 'arg2', 'kwarg1', **kwargs) - ... - - :param enforce_type: Enforce types of input parameters (Default=True) - :param returns: String describing the return values - :param rtype: String describing the data type of the return values - :param is_method: True if this is decorating an instance or class method, False otherwise (Default=True) - :param enforce_shape: Enforce the dimensions of input arrays (Default=True) - :param validator: :py:func:`dict` objects specifying the method parameters - :param options: additional options for documenting and validating method parameters - ''' - enforce_type = options.pop('enforce_type', True) - enforce_shape = options.pop('enforce_shape', True) - returns = options.pop('returns', None) - rtype = options.pop('rtype', None) - is_method = options.pop('is_method', True) - allow_extra = options.pop('allow_extra', False) - val_copy = __sort_args(_copy.deepcopy(validator)) - - def dec(func): - _docval = _copy.copy(options) - _docval['allow_extra'] = allow_extra - func.__name__ = _docval.get('func_name', func.__name__) - func.__doc__ = _docval.get('doc', func.__doc__) - pos = list() - kw = list() - for a in val_copy: - a['type'] = __resolve_type(a['type']) - if 'default' in a: - kw.append(a) - else: - pos.append(a) - loc_val = pos+kw - _docval[__docval_args_loc] = loc_val - if is_method: - def func_call(*args, **kwargs): - self = args[0] - parsed = __parse_args( - _copy.deepcopy(loc_val), - args[1:], - kwargs, - enforce_type=enforce_type, - enforce_shape=enforce_shape, - allow_extra=allow_extra) - - for error_type, ExceptionType in (('type_errors', TypeError), - ('value_errors', ValueError)): - parse_err = parsed.get(error_type) - if parse_err: - msg = ', '.join(parse_err) - raise_from(ExceptionType(msg), None) - - return func(self, **parsed['args']) - else: - def func_call(*args, **kwargs): - parsed = __parse_args(_copy.deepcopy(loc_val), - args, - kwargs, - enforce_type=enforce_type, - allow_extra=allow_extra) - for error_type, ExceptionType in (('type_errors', TypeError), - ('value_errors', ValueError)): - parse_err = parsed.get(error_type) - if parse_err: - msg = ', '.join(parse_err) - raise_from(ExceptionType(msg), None) - - return func(**parsed['args']) - _rtype = rtype - if isinstance(rtype, type): - _rtype = rtype.__name__ - docstring = __googledoc(func, _docval[__docval_args_loc], returns=returns, rtype=_rtype) - setattr(func_call, '__doc__', docstring) - setattr(func_call, '__name__', func.__name__) - setattr(func_call, docval_attr_name, _docval) - setattr(func_call, '__module__', func.__module__) - return func_call - return dec - - -def __sig_arg(argval): - if 'default' in argval: - default = argval['default'] - if isinstance(default, str): - default = "'%s'" % default - else: - default = str(default) - return "%s=%s" % (argval['name'], default) - else: - return argval['name'] - - -def __builddoc(func, validator, docstring_fmt, arg_fmt, ret_fmt=None, returns=None, rtype=None): - '''Generate a Spinxy docstring''' - def to_str(argtype): - if isinstance(argtype, type): - return argtype.__name__ - return argtype - - def __sphinx_arg(arg): - fmt = dict() - fmt['name'] = arg.get('name') - fmt['doc'] = arg.get('doc') - if isinstance(arg['type'], tuple) or isinstance(arg['type'], list): - fmt['type'] = " or ".join(map(to_str, arg['type'])) - else: - fmt['type'] = to_str(arg['type']) - return arg_fmt.format(**fmt) - - sig = "%s(%s)\n\n" % (func.__name__, ", ".join(map(__sig_arg, validator))) - desc = func.__doc__.strip() if func.__doc__ is not None else "" - sig += docstring_fmt.format(description=desc, args="\n".join(map(__sphinx_arg, validator))) - - if not (ret_fmt is None or returns is None or rtype is None): - sig += ret_fmt.format(returns=returns, rtype=rtype) - return sig - - -def __sphinxdoc(func, validator, returns=None, rtype=None): - arg_fmt = (":param {name}: {doc}\n" - ":type {name}: {type}") - docstring_fmt = ("{description}\n\n" - "{args}\n") - ret_fmt = (":returns: {returns}\n" - ":rtype: {rtype}") - return __builddoc(func, validator, docstring_fmt, arg_fmt, ret_fmt=ret_fmt, returns=returns, rtype=rtype) - - -def __googledoc(func, validator, returns=None, rtype=None): - arg_fmt = " {name} ({type}): {doc}" - docstring_fmt = "{description}\n\n" - if len(validator) > 0: - docstring_fmt += "Args:\n{args}\n" - ret_fmt = ("\nReturns:\n" - " {rtype}: {returns}") - return __builddoc(func, validator, docstring_fmt, arg_fmt, ret_fmt=ret_fmt, returns=returns, rtype=rtype) - - -def getargs(*argnames): - '''getargs(*argnames, argdict) - Convenience function to retrieve arguments from a dictionary in batch - ''' - if len(argnames) < 2: - raise ValueError('Must supply at least one key and a dict') - if not isinstance(argnames[-1], dict): - raise ValueError('last argument must be dict') - kwargs = argnames[-1] - if not argnames: - raise ValueError('must provide keyword to get') - if len(argnames) == 2: - return kwargs.get(argnames[0]) - return [kwargs.get(arg) for arg in argnames[:-1]] - - -def popargs(*argnames): - '''popargs(*argnames, argdict) - Convenience function to retrieve and remove arguments from a dictionary in batch - ''' - if len(argnames) < 2: - raise ValueError('Must supply at least one key and a dict') - if not isinstance(argnames[-1], dict): - raise ValueError('last argument must be dict') - kwargs = argnames[-1] - if not argnames: - raise ValueError('must provide keyword to pop') - if len(argnames) == 2: - return kwargs.pop(argnames[0]) - return [kwargs.pop(arg) for arg in argnames[:-1]] - - -class ExtenderMeta(ABCMeta): - """A metaclass that will extend the base class initialization - routine by executing additional functions defined in - classes that use this metaclass - - In general, this class should only be used by core developers. - """ - - __preinit = '__preinit' - - @classmethod - def pre_init(cls, func): - setattr(func, cls.__preinit, True) - return classmethod(func) - - __postinit = '__postinit' - - @classmethod - def post_init(cls, func): - '''A decorator for defining a routine to run after creation of a type object. - - An example use of this method would be to define a classmethod that gathers - any defined methods or attributes after the base Python type construction (i.e. after - :py:func:`type` has been called) - ''' - setattr(func, cls.__postinit, True) - return classmethod(func) - - def __init__(cls, name, bases, classdict): - it = (getattr(cls, n) for n in dir(cls)) - it = (a for a in it if hasattr(a, cls.__preinit)) - for func in it: - func(name, bases, classdict) - super(ExtenderMeta, cls).__init__(name, bases, classdict) - it = (getattr(cls, n) for n in dir(cls)) - it = (a for a in it if hasattr(a, cls.__postinit)) - for func in it: - func(name, bases, classdict) - - -def get_data_shape(data, strict_no_data_load=False): - """ - Helper function used to determine the shape of the given array. - - :param data: Array for which we should determine the shape. - :type data: List, numpy.ndarray, DataChunkIterator, any object that support __len__ or .shape. - :param strict_no_data_load: In order to determin the shape of nested tuples and lists, this function - recursively inspects elements along the dimensions, assuming that the data has a regular, - rectangular shape. In the case of out-of-core iterators this means that the first item - along each dimensions would potentially be loaded into memory. By setting this option - we enforce that this does not happen, at the cost that we may not be able to determine - the shape of the array. - :return: Tuple of ints indicating the size of known dimensions. Dimensions for which the size is unknown - will be set to None. - """ - def __get_shape_helper(local_data): - shape = list() - if hasattr(local_data, '__len__'): - shape.append(len(local_data)) - if len(local_data) and not isinstance(local_data[0], (text_type, binary_type)): - shape.extend(__get_shape_helper(local_data[0])) - return tuple(shape) - if hasattr(data, 'maxshape'): - return data.maxshape - if hasattr(data, 'shape'): - return data.shape - if hasattr(data, '__len__') and not isinstance(data, (text_type, binary_type)): - if not strict_no_data_load or (isinstance(data, list) or isinstance(data, tuple) or isinstance(data, set)): - return __get_shape_helper(data) - else: - return None - else: - return None - - -def pystr(s): - """ - Cross-version support for convertin a string of characters to Python str object - """ - if six.PY2 and isinstance(s, six.text_type): - return s.encode('ascii', 'ignore') - elif six.PY3 and isinstance(s, six.binary_type): - return s.decode('utf-8') - else: - return s diff --git a/src/pynwb/form/validate/__init__.py b/src/pynwb/form/validate/__init__.py deleted file mode 100644 index f167c4b86..000000000 --- a/src/pynwb/form/validate/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# flake8: noqa: F401 -from . import errors - -from .validator import ValidatorMap, Validator, AttributeValidator, DatasetValidator, GroupValidator -from .errors import * diff --git a/src/pynwb/form/validate/errors.py b/src/pynwb/form/validate/errors.py deleted file mode 100644 index 630468aac..000000000 --- a/src/pynwb/form/validate/errors.py +++ /dev/null @@ -1,144 +0,0 @@ - -from ..utils import docval, getargs -from ..spec.spec import DtypeHelper -from numpy import dtype - - -__all__ = [ - "Error", - "DtypeError", - "MissingError", - "ShapeError", - "MissingDataType", - "IllegalLinkError", - "IncorrectDataType" -] - - -class Error(object): - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'reason', 'type': str, 'doc': 'the reason for the error'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - self.__name = getargs('name', kwargs) - self.__reason = getargs('reason', kwargs) - self.__location = getargs('location', kwargs) - if self.__location is not None: - self.__str = "%s (%s): %s" % (self.__name, self.__location, self.__reason) - else: - self.__str = "%s: %s" % (self.name, self.reason) - - @property - def name(self): - return self.__name - - @property - def reason(self): - return self.__reason - - @property - def location(self): - return self.__location - - @location.setter - def location(self, loc): - self.__location = loc - self.__str = "%s (%s): %s" % (self.__name, self.__location, self.__reason) - - def __str__(self): - return self.__str - - def __repr__(self): - return self.__str__() - - -class DtypeError(Error): - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'expected', 'type': (dtype, type, str, list), 'doc': 'the expected dtype'}, - {'name': 'received', 'type': (dtype, type, str, list), 'doc': 'the received dtype'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - name = getargs('name', kwargs) - expected = getargs('expected', kwargs) - received = getargs('received', kwargs) - if isinstance(expected, list): - expected = DtypeHelper.simplify_cpd_type(expected) - reason = "incorrect type - expected '%s', got '%s'" % (expected, received) - loc = getargs('location', kwargs) - super(DtypeError, self).__init__(name, reason, location=loc) - - -class MissingError(Error): - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - name = getargs('name', kwargs) - reason = "argument missing" - loc = getargs('location', kwargs) - super(MissingError, self).__init__(name, reason, location=loc) - - -class MissingDataType(Error): - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'data_type', 'type': str, 'doc': 'the missing data type'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - name, data_type = getargs('name', 'data_type', kwargs) - self.__data_type = data_type - reason = "missing data type %s" % self.__data_type - loc = getargs('location', kwargs) - super(MissingDataType, self).__init__(name, reason, location=loc) - - @property - def data_type(self): - return self.__data_type - - -class ShapeError(Error): - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'expected', 'type': (tuple, list), 'doc': 'the expected shape'}, - {'name': 'received', 'type': (tuple, list), 'doc': 'the received shape'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - name = getargs('name', kwargs) - expected = getargs('expected', kwargs) - received = getargs('received', kwargs) - reason = "incorrect shape - expected '%s', got'%s'" % (expected, received) - loc = getargs('location', kwargs) - super(ShapeError, self).__init__(name, reason, location=loc) - - -class IllegalLinkError(Error): - """ - A validation error for indicating that a link was used where an actual object - (i.e. a dataset or a group) must be used - """ - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - name = getargs('name', kwargs) - reason = "illegal use of link" - loc = getargs('location', kwargs) - super(IllegalLinkError, self).__init__(name, reason, location=loc) - - -class IncorrectDataType(Error): - """ - A validation error for indicating that the incorrect data_type (not dtype) was used. - """ - - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'expected', 'type': str, 'doc': 'the expected data_type'}, - {'name': 'received', 'type': str, 'doc': 'the received data_type'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - name = getargs('name', kwargs) - expected = getargs('expected', kwargs) - received = getargs('received', kwargs) - reason = "incorrect data_type - expected '%s', got '%s'" % (expected, received) - loc = getargs('location', kwargs) - super(DtypeError, self).__init__(name, reason, location=loc) diff --git a/src/pynwb/form/validate/validator.py b/src/pynwb/form/validate/validator.py deleted file mode 100644 index 032766ac6..000000000 --- a/src/pynwb/form/validate/validator.py +++ /dev/null @@ -1,468 +0,0 @@ -import numpy as np -from abc import ABCMeta, abstractmethod -from copy import copy -import re -from itertools import chain - -from ..utils import docval, getargs, call_docval_func, pystr -from ..data_utils import get_shape - -from ..spec import Spec, AttributeSpec, GroupSpec, DatasetSpec, RefSpec -from ..spec.spec import BaseStorageSpec, DtypeHelper -from ..spec import SpecNamespace - -from ..build import GroupBuilder, DatasetBuilder, LinkBuilder, ReferenceBuilder, RegionBuilder -from ..build.builders import BaseBuilder - -from .errors import Error, DtypeError, MissingError, MissingDataType, ShapeError, IllegalLinkError, IncorrectDataType -from six import with_metaclass, raise_from, text_type, binary_type - - -__synonyms = DtypeHelper.primary_dtype_synonyms - -__additional = { - 'float': ['double'], - 'int8': ['short', 'int', 'long'], - 'short': ['int', 'long'], - 'int': ['long'], - 'uint8': ['uint16', 'uint32', 'uint64'], - 'uint16': ['uint32', 'uint64'], - 'uint32': ['uint64'], -} - -__allowable = dict() -for dt, dt_syn in __synonyms.items(): - allow = copy(dt_syn) - if dt in __additional: - for addl in __additional[dt]: - allow.extend(__synonyms[addl]) - for syn in dt_syn: - __allowable[syn] = allow -__allowable['numeric'] = set(chain.from_iterable(__allowable[k] for k in __allowable if 'int' in k or 'float' in k)) - - -def check_type(expected, received): - ''' - *expected* should come from the spec - *received* should come from the data - ''' - if isinstance(expected, list): - if len(expected) > len(received): - raise ValueError('compound type shorter than expected') - for i, exp in enumerate(DtypeHelper.simplify_cpd_type(expected)): - rec = received[i] - if rec not in __allowable[exp]: - return False - return True - else: - if isinstance(received, np.dtype): - if received.char == 'O': - if 'vlen' in received.metadata: - received = received.metadata['vlen'] - else: - raise ValueError("Unrecognized type: '%s'" % received) - received = 'utf' if received is text_type else 'ascii' - elif received.char == 'U': - received = 'utf' - elif received.char == 'S': - received = 'ascii' - else: - received = received.name - elif isinstance(received, type): - received = received.__name__ - if isinstance(expected, RefSpec): - expected = expected.reftype - elif isinstance(expected, type): - expected = expected.__name__ - return received in __allowable[expected] - - -def get_iso8601_regex(): - isodate_re = (r'^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):' - r'([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-](?:2[0-3]|[01][0-9]):[0-5][0-9])?$') - return re.compile(isodate_re) - - -_iso_re = get_iso8601_regex() - - -def _check_isodatetime(s, default=None): - try: - if _iso_re.match(pystr(s)) is not None: - return 'isodatetime' - except Exception: - pass - return default - - -def get_type(data): - if isinstance(data, text_type): - return _check_isodatetime(data, 'utf') - elif isinstance(data, binary_type): - return _check_isodatetime(data, 'ascii') - elif isinstance(data, RegionBuilder): - return 'region' - elif isinstance(data, ReferenceBuilder): - return 'object' - elif isinstance(data, np.ndarray): - return get_type(data[0]) - if not hasattr(data, '__len__'): - return type(data).__name__ - else: - if hasattr(data, 'dtype'): - return data.dtype - if len(data) == 0: - raise ValueError('cannot determine type for empty data') - return get_type(data[0]) - - -def check_shape(expected, received): - ret = False - if expected is None: - ret = True - else: - if isinstance(expected, (list, tuple)): - if isinstance(expected[0], (list, tuple)): - for sub in expected: - if check_shape(sub, received): - ret = True - break - else: - if len(expected) == len(received): - ret = True - for e, r in zip(expected, received): - if not check_shape(e, r): - ret = False - break - elif isinstance(expected, int): - ret = expected == received - return ret - - -class ValidatorMap(object): - """A class for keeping track of Validator objects for all data types in a namespace""" - - @docval({'name': 'namespace', 'type': SpecNamespace, 'doc': 'the namespace to builder map for'}) - def __init__(self, **kwargs): - ns = getargs('namespace', kwargs) - self.__ns = ns - tree = dict() - types = ns.get_registered_types() - self.__type_key = ns.get_spec(types[0]).type_key() - for dt in types: - spec = ns.get_spec(dt) - parent = spec.data_type_inc - child = spec.data_type_def - tree.setdefault(child, list()) - if parent is not None: - tree.setdefault(parent, list()).append(child) - for t in tree: - self.__rec(tree, t) - self.__valid_types = dict() - self.__validators = dict() - for dt, children in tree.items(): - _list = list() - for t in children: - spec = self.__ns.get_spec(t) - if isinstance(spec, GroupSpec): - val = GroupValidator(spec, self) - else: - val = DatasetValidator(spec, self) - if t == dt: - self.__validators[t] = val - _list.append(val) - self.__valid_types[dt] = tuple(_list) - - def __rec(self, tree, node): - if isinstance(tree[node], tuple): - return tree[node] - sub_types = {node} - for child in tree[node]: - sub_types.update(self.__rec(tree, child)) - tree[node] = tuple(sub_types) - return tree[node] - - @property - def namespace(self): - return self.__ns - - @docval({'name': 'spec', 'type': (Spec, str), 'doc': 'the specification to use to validate'}, - returns='all valid sub data types for the given spec', rtype=tuple) - def valid_types(self, **kwargs): - '''Get all valid types for a given data type''' - spec = getargs('spec', kwargs) - if isinstance(spec, Spec): - spec = spec.data_type_def - try: - return self.__valid_types[spec] - except KeyError: - raise_from(ValueError("no children for '%s'" % spec), None) - - @docval({'name': 'data_type', 'type': (BaseStorageSpec, str), - 'doc': 'the data type to get the validator for'}, - returns='the validator ``data_type``') - def get_validator(self, **kwargs): - """Return the validator for a given data type""" - dt = getargs('data_type', kwargs) - if isinstance(dt, BaseStorageSpec): - dt_tmp = dt.data_type_def - if dt_tmp is None: - dt_tmp = dt.data_type_inc - dt = dt_tmp - try: - return self.__validators[dt] - except KeyError: - msg = "data type '%s' not found in namespace %s" % (dt, self.__ns.name) - raise_from(ValueError(msg), None) - - @docval({'name': 'builder', 'type': BaseBuilder, 'doc': 'the builder to validate'}, - returns="a list of errors found", rtype=list) - def validate(self, **kwargs): - """Validate a builder against a Spec - - ``builder`` must have the attribute used to specifying data type - by the namespace used to construct this ValidatorMap. - """ - builder = getargs('builder', kwargs) - dt = builder.attributes.get(self.__type_key) - if dt is None: - msg = "builder must have data type defined with attribute '%s'" % self.__type_key - raise ValueError(msg) - validator = self.get_validator(dt) - return validator.validate(builder) - - -class Validator(with_metaclass(ABCMeta, object)): - '''A base class for classes that will be used to validate against Spec subclasses''' - - @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to use to validate'}, - {'name': 'validator_map', 'type': ValidatorMap, 'doc': 'the ValidatorMap to use during validation'}) - def __init__(self, **kwargs): - self.__spec = getargs('spec', kwargs) - self.__vmap = getargs('validator_map', kwargs) - - @property - def spec(self): - return self.__spec - - @property - def vmap(self): - return self.__vmap - - @abstractmethod - @docval({'name': 'value', 'type': None, 'doc': 'either in the form of a value or a Builder'}, - returns='a list of Errors', rtype=list) - def validate(self, **kwargs): - pass - - @classmethod - def get_spec_loc(cls, spec): - stack = list() - tmp = spec - while tmp is not None: - name = tmp.name - if name is None: - name = tmp.data_type_def - if name is None: - name = tmp.data_type_inc - stack.append(name) - tmp = tmp.parent - return "/".join(reversed(stack)) - - @classmethod - def get_builder_loc(cls, builder): - stack = list() - tmp = builder - while tmp is not None and tmp.name != 'root': - stack.append(tmp.name) - tmp = tmp.parent - return "/".join(reversed(stack)) - - -class AttributeValidator(Validator): - '''A class for validating values against AttributeSpecs''' - - @docval({'name': 'spec', 'type': AttributeSpec, 'doc': 'the specification to use to validate'}, - {'name': 'validator_map', 'type': ValidatorMap, 'doc': 'the ValidatorMap to use during validation'}) - def __init__(self, **kwargs): - call_docval_func(super(AttributeValidator, self).__init__, kwargs) - - @docval({'name': 'value', 'type': None, 'doc': 'the value to validate'}, - returns='a list of Errors', rtype=list) - def validate(self, **kwargs): - value = getargs('value', kwargs) - ret = list() - spec = self.spec - if spec.required and value is None: - ret.append(MissingError(self.get_spec_loc(spec))) - else: - if spec.dtype is None: - ret.append(Error(self.get_spec_loc(spec))) - elif isinstance(spec.dtype, RefSpec): - if not isinstance(value, BaseBuilder): - expected = '%s reference' % spec.dtype.reftype - ret.append(DtypeError(self.get_spec_loc(spec), expected, get_type(value))) - else: - target_spec = self.vmap.namespace.catalog.get_spec(spec.dtype.target_type) - data_type = value.attributes.get(target_spec.type_key()) - hierarchy = self.vmap.namespace.catalog.get_hierarchy(data_type) - if spec.dtype.target_type not in hierarchy: - ret.append(IncorrectDataType(self.get_spec_loc(spec), spec.dtype.target_type, data_type)) - else: - dtype = get_type(value) - if not check_type(spec.dtype, dtype): - ret.append(DtypeError(self.get_spec_loc(spec), spec.dtype, dtype)) - shape = get_shape(value) - if not check_shape(spec.shape, shape): - ret.append(ShapeError(self.get_spec_loc(spec), spec.shape, shape)) - return ret - - -class BaseStorageValidator(Validator): - '''A base class for validating against Spec objects that have attributes i.e. BaseStorageSpec''' - - @docval({'name': 'spec', 'type': BaseStorageSpec, 'doc': 'the specification to use to validate'}, - {'name': 'validator_map', 'type': ValidatorMap, 'doc': 'the ValidatorMap to use during validation'}) - def __init__(self, **kwargs): - call_docval_func(super(BaseStorageValidator, self).__init__, kwargs) - self.__attribute_validators = dict() - for attr in self.spec.attributes: - self.__attribute_validators[attr.name] = AttributeValidator(attr, self.vmap) - - @docval({"name": "builder", "type": BaseBuilder, "doc": "the builder to validate"}, - returns='a list of Errors', rtype=list) - def validate(self, **kwargs): - builder = getargs('builder', kwargs) - attributes = builder.attributes - ret = list() - for attr, validator in self.__attribute_validators.items(): - attr_val = attributes.get(attr) - if attr_val is None: - if validator.spec.required: - ret.append(MissingError(self.get_spec_loc(validator.spec), - location=self.get_builder_loc(builder))) - else: - errors = validator.validate(attr_val) - for err in errors: - err.location = self.get_builder_loc(builder) + ".%s" % validator.spec.name - ret.extend(errors) - return ret - - -class DatasetValidator(BaseStorageValidator): - '''A class for validating DatasetBuilders against DatasetSpecs''' - - @docval({'name': 'spec', 'type': DatasetSpec, 'doc': 'the specification to use to validate'}, - {'name': 'validator_map', 'type': ValidatorMap, 'doc': 'the ValidatorMap to use during validation'}) - def __init__(self, **kwargs): - call_docval_func(super(DatasetValidator, self).__init__, kwargs) - - @docval({"name": "builder", "type": DatasetBuilder, "doc": "the builder to validate"}, - returns='a list of Errors', rtype=list) - def validate(self, **kwargs): - builder = getargs('builder', kwargs) - ret = super(DatasetValidator, self).validate(builder) - data = builder.data - if self.spec.dtype is not None: - dtype = get_type(data) - if not check_type(self.spec.dtype, dtype): - ret.append(DtypeError(self.get_spec_loc(self.spec), self.spec.dtype, dtype, - location=self.get_builder_loc(builder))) - shape = get_shape(data) - if not check_shape(self.spec.shape, shape): - ret.append(ShapeError(self.get_spec_loc(self.spec), self.spec.shape, shape, - location=self.get_builder_loc(builder))) - return ret - - -class GroupValidator(BaseStorageValidator): - '''A class for validating GroupBuilders against GroupSpecs''' - - @docval({'name': 'spec', 'type': GroupSpec, 'doc': 'the specification to use to validate'}, - {'name': 'validator_map', 'type': ValidatorMap, 'doc': 'the ValidatorMap to use during validation'}) - def __init__(self, **kwargs): - call_docval_func(super(GroupValidator, self).__init__, kwargs) - self.__include_dts = dict() - self.__dataset_validators = dict() - self.__group_validators = dict() - it = chain(self.spec.datasets, self.spec.groups) - for spec in it: - if spec.data_type_def is None: - if spec.data_type_inc is None: - if isinstance(spec, GroupSpec): - self.__group_validators[spec.name] = GroupValidator(spec, self.vmap) - else: - self.__dataset_validators[spec.name] = DatasetValidator(spec, self.vmap) - else: - self.__include_dts[spec.data_type_inc] = spec - else: - self.__include_dts[spec.data_type_def] = spec - - @docval({"name": "builder", "type": GroupBuilder, "doc": "the builder to validate"}, # noqa: C901 - returns='a list of Errors', rtype=list) - def validate(self, **kwargs): - builder = getargs('builder', kwargs) - ret = super(GroupValidator, self).validate(builder) - # get the data_types - data_types = dict() - for key, value in builder.items(): - v_builder = value - if isinstance(v_builder, LinkBuilder): - v_builder = v_builder.builder - if isinstance(v_builder, BaseBuilder): - dt = v_builder.attributes.get(self.spec.type_key()) - if dt is not None: - data_types.setdefault(dt, list()).append(value) - for dt, inc_spec in self.__include_dts.items(): - found = False - inc_name = inc_spec.name - for sub_val in self.vmap.valid_types(dt): - spec = sub_val.spec - sub_dt = spec.data_type_def - dt_builders = data_types.get(sub_dt) - if dt_builders is not None: - if inc_name is not None: - dt_builders = filter(lambda x: x.name == inc_name, dt_builders) # noqa: F405 - for bldr in dt_builders: - tmp = bldr - if isinstance(bldr, LinkBuilder): - if inc_spec.linkable: - tmp = bldr.builder - else: - ret.append(IllegalLinkError(self.get_spec_loc(inc_spec), - location=self.get_builder_loc(tmp))) - ret.extend(sub_val.validate(tmp)) - found = True - if not found and self.__include_dts[dt].required: - ret.append(MissingDataType(self.get_spec_loc(self.spec), dt, - location=self.get_builder_loc(builder))) - it = chain(self.__dataset_validators.items(), - self.__group_validators.items()) - for name, validator in it: - sub_builder = builder.get(name) - if isinstance(validator, BaseStorageSpec): - inc_spec = validator - validator = self.vmap.get_validator(inc_spec) - def_spec = validator.spec - if sub_builder is None: - if inc_spec.required: - ret.append(MissingDataType(self.get_spec_loc(def_spec), def_spec.data_type_def, - location=self.get_builder_loc(builder))) - else: - ret.extend(validator.validate(sub_builder)) - - else: - spec = validator.spec - if isinstance(sub_builder, LinkBuilder): - if spec.linkable: - sub_builder = sub_builder.builder - else: - ret.append(IllegalLinkError(self.get_spec_loc(spec), location=self.get_builder_loc(builder))) - continue - if sub_builder is None: - if spec.required: - ret.append(MissingError(self.get_spec_loc(spec), location=self.get_builder_loc(builder))) - else: - ret.extend(validator.validate(sub_builder)) - - return ret diff --git a/src/pynwb/icephys.py b/src/pynwb/icephys.py index f539626b2..146ab9ff8 100644 --- a/src/pynwb/icephys.py +++ b/src/pynwb/icephys.py @@ -2,7 +2,7 @@ import numpy as np -from .form.utils import docval, popargs, fmt_docval_args, call_docval_func +from hdmf.utils import docval, popargs, fmt_docval_args, call_docval_func from . import register_class, CORE_NAMESPACE from .base import TimeSeries, _default_resolution, _default_conversion diff --git a/src/pynwb/image.py b/src/pynwb/image.py index bd80ee464..38aa59d18 100644 --- a/src/pynwb/image.py +++ b/src/pynwb/image.py @@ -2,7 +2,7 @@ import warnings from collections import Iterable -from .form.utils import docval, popargs, call_docval_func +from hdmf.utils import docval, popargs, call_docval_func from . import register_class, CORE_NAMESPACE from .base import TimeSeries, _default_resolution, _default_conversion, Image diff --git a/src/pynwb/io/base.py b/src/pynwb/io/base.py index 1d7315479..781d202a2 100644 --- a/src/pynwb/io/base.py +++ b/src/pynwb/io/base.py @@ -2,7 +2,7 @@ from .. import register_map from ..base import TimeSeries, ProcessingModule -from ..form.build import LinkBuilder +from hdmf.build import LinkBuilder @register_map(ProcessingModule) diff --git a/src/pynwb/io/core.py b/src/pynwb/io/core.py index 30293b4fa..c1371f5b7 100644 --- a/src/pynwb/io/core.py +++ b/src/pynwb/io/core.py @@ -1,7 +1,7 @@ -from ..form.utils import docval, getargs -from ..form.build import ObjectMapper, RegionBuilder, BuildManager -from ..form.spec import Spec -from ..form.container import Container +from hdmf.utils import docval, getargs +from hdmf.build import ObjectMapper, RegionBuilder, BuildManager +from hdmf.spec import Spec +from hdmf.container import Container from .. import register_map from pynwb.file import NWBFile diff --git a/src/pynwb/io/file.py b/src/pynwb/io/file.py index a58705af7..05a944e8d 100644 --- a/src/pynwb/io/file.py +++ b/src/pynwb/io/file.py @@ -1,5 +1,5 @@ from dateutil.parser import parse as dateutil_parse -from ..form.build import ObjectMapper +from hdmf.build import ObjectMapper from .. import register_map from ..file import NWBFile diff --git a/src/pynwb/legacy/__init__.py b/src/pynwb/legacy/__init__.py index a7e0519d7..4c03eef79 100644 --- a/src/pynwb/legacy/__init__.py +++ b/src/pynwb/legacy/__init__.py @@ -1,6 +1,6 @@ -from ..form.spec import NamespaceCatalog -from ..form.utils import docval, getargs +from hdmf.spec import NamespaceCatalog +from hdmf.utils import docval, getargs from ..spec import NWBDatasetSpec, NWBGroupSpec, NWBNamespace from .. import _get_resources, get_type_map, NWBContainer diff --git a/src/pynwb/legacy/map.py b/src/pynwb/legacy/map.py index 1b1dde53e..a6fa22461 100644 --- a/src/pynwb/legacy/map.py +++ b/src/pynwb/legacy/map.py @@ -1,6 +1,6 @@ -from ..form.build.map import ObjectMapper, TypeMap -from ..form.build.builders import GroupBuilder +from hdmf.build.map import ObjectMapper, TypeMap +from hdmf.build.builders import GroupBuilder def decode(val): diff --git a/src/pynwb/misc.py b/src/pynwb/misc.py index 790dc7f94..d08715786 100644 --- a/src/pynwb/misc.py +++ b/src/pynwb/misc.py @@ -1,7 +1,7 @@ import numpy as np from collections import Iterable -from .form.utils import docval, getargs, popargs, call_docval_func +from hdmf.utils import docval, getargs, popargs, call_docval_func from . import register_class, CORE_NAMESPACE from .base import TimeSeries, _default_conversion, _default_resolution diff --git a/src/pynwb/ogen.py b/src/pynwb/ogen.py index f5cdfb729..f6373fc21 100644 --- a/src/pynwb/ogen.py +++ b/src/pynwb/ogen.py @@ -1,6 +1,6 @@ from collections import Iterable -from .form.utils import docval, popargs, fmt_docval_args +from hdmf.utils import docval, popargs, fmt_docval_args from . import register_class, CORE_NAMESPACE from .base import TimeSeries, _default_resolution, _default_conversion diff --git a/src/pynwb/ophys.py b/src/pynwb/ophys.py index b898fe2e1..e76c7c8ce 100644 --- a/src/pynwb/ophys.py +++ b/src/pynwb/ophys.py @@ -1,7 +1,7 @@ from collections import Iterable import numpy as np -from .form.utils import docval, getargs, popargs, fmt_docval_args, call_docval_func +from hdmf.utils import docval, getargs, popargs, fmt_docval_args, call_docval_func from . import register_class, CORE_NAMESPACE from .base import TimeSeries, _default_resolution, _default_conversion diff --git a/src/pynwb/retinotopy.py b/src/pynwb/retinotopy.py index 96220943e..5d024797e 100644 --- a/src/pynwb/retinotopy.py +++ b/src/pynwb/retinotopy.py @@ -1,6 +1,6 @@ from collections import Iterable -from .form.utils import docval, popargs, fmt_docval_args +from hdmf.utils import docval, popargs, fmt_docval_args from . import register_class, CORE_NAMESPACE from .core import NWBContainer, NWBDataInterface diff --git a/src/pynwb/spec.py b/src/pynwb/spec.py index 1fc3f7397..e248469a4 100644 --- a/src/pynwb/spec.py +++ b/src/pynwb/spec.py @@ -1,8 +1,8 @@ from copy import copy, deepcopy -from .form.spec import LinkSpec, GroupSpec, DatasetSpec, SpecNamespace,\ +from hdmf.spec import LinkSpec, GroupSpec, DatasetSpec, SpecNamespace,\ NamespaceBuilder, AttributeSpec, DtypeSpec, RefSpec -from .form.utils import docval, get_docval, fmt_docval_args +from hdmf.utils import docval, get_docval, fmt_docval_args from . import CORE_NAMESPACE diff --git a/test.py b/test.py index 5898b393e..bd2bb2317 100755 --- a/test.py +++ b/test.py @@ -14,7 +14,7 @@ import traceback import unittest2 as unittest -flags = {'form': 1, 'pynwb': 2, 'integration': 3, 'example': 4} +flags = {'pynwb': 2, 'integration': 3, 'example': 4} TOTAL = 0 FAILURES = 0 @@ -180,8 +180,6 @@ def main(): parser.set_defaults(verbosity=1, suites=[]) parser.add_argument('-v', '--verbose', const=2, dest='verbosity', action='store_const', help='run in verbose mode') parser.add_argument('-q', '--quiet', const=0, dest='verbosity', action='store_const', help='run disabling output') - parser.add_argument('-f', '--form', action='append_const', const=flags['form'], dest='suites', - help='run unit tests for form package') parser.add_argument('-p', '--pynwb', action='append_const', const=flags['pynwb'], dest='suites', help='run unit tests for pynwb package') parser.add_argument('-i', '--integration', action='append_const', const=flags['integration'], dest='suites', @@ -203,10 +201,6 @@ def main(): ch.setFormatter(formatter) root.addHandler(ch) - # Run unit tests for form package - if flags['form'] in args.suites: - run_test_suite("tests/unit/form_tests", "form unit tests", verbose=args.verbosity) - # Run unit tests for pynwb package if flags['pynwb'] in args.suites: run_test_suite("tests/unit/pynwb_tests", "pynwb unit tests", verbose=args.verbosity) diff --git a/tests/integration/test_io.py b/tests/integration/test_io.py index 708797686..d08802324 100644 --- a/tests/integration/test_io.py +++ b/tests/integration/test_io.py @@ -7,10 +7,10 @@ from pynwb import NWBFile, TimeSeries, get_manager, NWBHDF5IO -from pynwb.form.backends.hdf5 import HDF5IO, H5DataIO -from pynwb.form.data_utils import DataChunkIterator -from pynwb.form.build import GroupBuilder, DatasetBuilder -from pynwb.form.spec import NamespaceCatalog +from hdmf.backends.hdf5 import HDF5IO, H5DataIO +from hdmf.data_utils import DataChunkIterator +from hdmf.build import GroupBuilder, DatasetBuilder +from hdmf.spec import NamespaceCatalog from pynwb.spec import NWBGroupSpec, NWBDatasetSpec, NWBNamespace from pynwb.ecephys import ElectricalSeries, LFP diff --git a/tests/integration/ui_write/base.py b/tests/integration/ui_write/base.py index c0f88858d..7aa7617b6 100644 --- a/tests/integration/ui_write/base.py +++ b/tests/integration/ui_write/base.py @@ -6,7 +6,7 @@ import numpy.testing as npt from pynwb import NWBContainer, get_manager, NWBFile, NWBData -from pynwb.form.backends.hdf5 import HDF5IO +from hdmf.backends.hdf5 import HDF5IO CORE_NAMESPACE = 'core' diff --git a/tests/integration/ui_write/test_base.py b/tests/integration/ui_write/test_base.py index d5898b7e0..91b61bbfd 100644 --- a/tests/integration/ui_write/test_base.py +++ b/tests/integration/ui_write/test_base.py @@ -1,7 +1,7 @@ import numpy as np import datetime as datetime -from pynwb.form.build import GroupBuilder, DatasetBuilder +from hdmf.build import GroupBuilder, DatasetBuilder from pynwb import TimeSeries, NWBFile, NWBHDF5IO diff --git a/tests/integration/ui_write/test_ecephys.py b/tests/integration/ui_write/test_ecephys.py index b218a03aa..39ace34e5 100644 --- a/tests/integration/ui_write/test_ecephys.py +++ b/tests/integration/ui_write/test_ecephys.py @@ -1,6 +1,6 @@ import unittest2 as unittest -from pynwb.form.build import GroupBuilder, DatasetBuilder, LinkBuilder, ReferenceBuilder +from hdmf.build import GroupBuilder, DatasetBuilder, LinkBuilder, ReferenceBuilder from pynwb.ecephys import ElectrodeGroup, ElectricalSeries, FilteredEphys, LFP, Clustering, ClusterWaveforms,\ SpikeEventSeries, EventWaveform, EventDetection, FeatureExtraction diff --git a/tests/integration/ui_write/test_icephys.py b/tests/integration/ui_write/test_icephys.py index 057e6bcfb..380bac797 100644 --- a/tests/integration/ui_write/test_icephys.py +++ b/tests/integration/ui_write/test_icephys.py @@ -1,4 +1,4 @@ -from pynwb.form.build import GroupBuilder, LinkBuilder, ReferenceBuilder, DatasetBuilder +from hdmf.build import GroupBuilder, LinkBuilder, ReferenceBuilder, DatasetBuilder from pynwb import NWBFile from pynwb.icephys import (IntracellularElectrode, PatchClampSeries, CurrentClampStimulusSeries, SweepTable, diff --git a/tests/integration/ui_write/test_misc.py b/tests/integration/ui_write/test_misc.py index f50d89c44..aab30b62b 100644 --- a/tests/integration/ui_write/test_misc.py +++ b/tests/integration/ui_write/test_misc.py @@ -1,7 +1,7 @@ from . import base import numpy as np -from pynwb.form.build import GroupBuilder, DatasetBuilder, ReferenceBuilder +from hdmf.build import GroupBuilder, DatasetBuilder, ReferenceBuilder from pynwb import TimeSeries from pynwb.core import DynamicTable, VectorData diff --git a/tests/integration/ui_write/test_nwbfile.py b/tests/integration/ui_write/test_nwbfile.py index 5e4463596..04b4669f8 100644 --- a/tests/integration/ui_write/test_nwbfile.py +++ b/tests/integration/ui_write/test_nwbfile.py @@ -5,8 +5,8 @@ import pandas as pd import numpy as np -from pynwb.form.build import GroupBuilder, DatasetBuilder -from pynwb.form.backends.hdf5 import HDF5IO +from hdmf.build import GroupBuilder, DatasetBuilder +from hdmf.backends.hdf5 import HDF5IO from pynwb import NWBFile, TimeSeries from pynwb.file import Subject diff --git a/tests/integration/ui_write/test_ophys.py b/tests/integration/ui_write/test_ophys.py index 5a9ed1464..d8e305e74 100644 --- a/tests/integration/ui_write/test_ophys.py +++ b/tests/integration/ui_write/test_ophys.py @@ -2,7 +2,7 @@ import numpy as np from copy import deepcopy -from pynwb.form.build import GroupBuilder, DatasetBuilder, LinkBuilder, ReferenceBuilder +from hdmf.build import GroupBuilder, DatasetBuilder, LinkBuilder, ReferenceBuilder from pynwb.ophys import ( ImagingPlane, diff --git a/tests/unit/form_tests/__init__.py b/tests/unit/form_tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/unit/form_tests/build_tests/__init__.py b/tests/unit/form_tests/build_tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/unit/form_tests/build_tests/test_io_build_builders.py b/tests/unit/form_tests/build_tests/test_io_build_builders.py deleted file mode 100644 index 73cc0feac..000000000 --- a/tests/unit/form_tests/build_tests/test_io_build_builders.py +++ /dev/null @@ -1,313 +0,0 @@ -import unittest2 as unittest - -from pynwb.form.build import GroupBuilder, DatasetBuilder, LinkBuilder - - -class GroupBuilderSetterTests(unittest.TestCase): - """Tests for setter functions in GroupBuilder class""" - - def setUp(self): - self.gb = GroupBuilder('gb') - self.gb2 = GroupBuilder('gb2', source='file1') - - def tearDown(self): - pass - - def test_setitem_disabled(self): - """Test __set_item__ is disabled""" - with self.assertRaises(NotImplementedError): - self.gb['key'] = 'value' - - def test_add_dataset(self): - ds = self.gb.add_dataset('my_dataset', list(range(10))) - self.assertIsInstance(ds, DatasetBuilder) - self.assertIs(self.gb, ds.parent) - - def test_add_group(self): - gp = self.gb.add_group('my_subgroup') - self.assertIsInstance(gp, GroupBuilder) - self.assertIs(self.gb['my_subgroup'], gp) - self.assertIs(self.gb, gp.parent) - - def test_add_link(self): - gp = self.gb.add_group('my_subgroup') - sl = self.gb.add_link(gp, 'my_link') - self.assertIsInstance(sl, LinkBuilder) - self.assertIs(self.gb['my_link'], sl) - self.assertIs(self.gb, sl.parent) - - def test_add_external_link(self): - gp = self.gb2.add_group('my_subgroup') - el = self.gb.add_link(gp, 'my_externallink') - self.assertIsInstance(el, LinkBuilder) - self.assertIs(self.gb['my_externallink'], el) - self.assertIs(self.gb, el.parent) - self.assertIs(self.gb2, gp.parent) - - # @unittest.expectedFailure - def test_set_attribute(self): - self.gb.set_attribute('key', 'value') - self.assertIn('key', self.gb.obj_type) - # self.assertEqual(dict.__getitem__(self.gb, 'attributes')['key'], 'value') - self.assertEqual(self.gb['key'], 'value') - - def test_parent_constructor(self): - gb2 = GroupBuilder('gb2', parent=self.gb) - self.assertIs(gb2.parent, self.gb) - - def test_set_group(self): - self.gb.set_group(self.gb2) - self.assertIs(self.gb2.parent, self.gb) - - -class GroupBuilderGetterTests(unittest.TestCase): - - def setUp(self): - self.subgroup1 = GroupBuilder('subgroup1') - self.dataset1 = DatasetBuilder('dataset1', list(range(10))) - self.soft_link1 = LinkBuilder(self.subgroup1, 'soft_link1') - self.int_attr = 1 - self.str_attr = "my_str" - - self.group1 = GroupBuilder('group1', {'subgroup1': self.subgroup1}) - self.gb = GroupBuilder('gb', {'group1': self.group1}, - {'dataset1': self.dataset1}, - {'int_attr': self.int_attr, - 'str_attr': self.str_attr}, - {'soft_link1': self.soft_link1}) - # {'soft_link1': self.soft_link1, - # 'external_link1': self.external_link1})) - - def tearDown(self): - pass - - def test_path(self): - self.assertEquals(self.subgroup1.path, 'gb/group1/subgroup1') - self.assertEquals(self.dataset1.path, 'gb/dataset1') - self.assertEquals(self.soft_link1.path, 'gb/soft_link1') - self.assertEquals(self.group1.path, 'gb/group1') - self.assertEquals(self.gb.path, 'gb') - - def test_get_item_group(self): - """Test __get_item__ for groups""" - self.assertIs(self.gb['group1'], self.group1) - - def test_get_item_group_subgroup1(self): - """Test __get_item__ for groups deeper in hierarchy""" - self.assertIs(self.gb['group1/subgroup1'], self.subgroup1) - - def test_get_item_dataset(self): - """Test __get_item__ for datasets""" - self.assertIs(self.gb['dataset1'], self.dataset1) - - def test_get_item_attr1(self): - """Test __get_item__ for attributes""" - self.assertEqual(self.gb['int_attr'], self.int_attr) - - def test_get_item_attr2(self): - """Test __get_item__ for attributes""" - self.assertEqual(self.gb['str_attr'], self.str_attr) - - def test_get_item_invalid_key(self): - """Test __get_item__ for invalid key""" - with self.assertRaises(KeyError): - self.gb['invalid_key'] - - def test_get_item_soft_link(self): - """Test __get_item__ for soft links""" - self.assertIs(self.gb['soft_link1'], self.soft_link1) - - def test_get_group(self): - """Test get() for groups""" - self.assertIs(self.gb.get('group1'), self.group1) - - def test_get_group_subgroup1(self): - """Test get() for groups deeper in hierarchy""" - self.assertIs(self.gb.get('group1/subgroup1'), self.subgroup1) - - def test_get_dataset(self): - """Test get() for datasets""" - self.assertIs(self.gb.get('dataset1'), self.dataset1) - - def test_get_attr1(self): - """Test get() for attributes""" - self.assertEqual(self.gb.get('int_attr'), self.int_attr) - - def test_get_attr2(self): - """Test get() for attributes""" - self.assertEqual(self.gb.get('str_attr'), self.str_attr) - - def test_get_item_soft_link(self): # noqa: F811 - """Test get() for soft links""" - self.assertIs(self.gb.get('soft_link1'), self.soft_link1) - - def test_get_invalid_key(self): - """Test get() for invalid key""" - self.assertIs(self.gb.get('invalid_key'), None) - - def test_items(self): - """Test items()""" - items = ( - ('group1', self.group1), - ('dataset1', self.dataset1), - ('int_attr', self.int_attr), - ('str_attr', self.str_attr), - ('soft_link1', self.soft_link1), - # ('external_link1', self.external_link1) - ) - # self.assertSetEqual(items, set(self.gb.items())) - try: - self.assertCountEqual(items, self.gb.items()) - except AttributeError: - self.assertItemsEqual(items, self.gb.items()) - - def test_keys(self): - """Test keys()""" - keys = ( - 'group1', - 'dataset1', - 'int_attr', - 'str_attr', - 'soft_link1', - # 'external_link1', - ) - try: - self.assertCountEqual(keys, self.gb.keys()) - except AttributeError: - self.assertItemsEqual(keys, self.gb.keys()) - - def test_values(self): - """Test values()""" - values = ( - self.group1, - self.dataset1, - self.int_attr, self.str_attr, - self.soft_link1, - # self.external_link1, - ) - try: - self.assertCountEqual(values, self.gb.values()) - except AttributeError: - self.assertItemsEqual(values, self.gb.values()) - - -class GroupBuilderIsEmptyTests(unittest.TestCase): - - def test_is_empty_true(self): - """Test empty when group has nothing in it""" - gb = GroupBuilder('gb') - self.assertEqual(gb.is_empty(), True) - - def test_is_empty_true_group(self): - """Test is_empty() when group has an empty subgroup""" - gb = GroupBuilder('gb', {'my_subgroup': GroupBuilder('my_subgroup')}) - self.assertEqual(gb.is_empty(), True) - - def test_is_empty_false_dataset(self): - """Test is_empty() when group has a dataset""" - gb = GroupBuilder('gb', datasets={'my_dataset': DatasetBuilder('my_dataset')}) - self.assertEqual(gb.is_empty(), False) - - def test_is_empty_false_group_dataset(self): - """Test is_empty() when group has a subgroup with a dataset""" - gb = GroupBuilder( - 'gb', - {'my_subgroup': - GroupBuilder( - 'my_subgroup', - datasets={'my_dataset': DatasetBuilder('my_dataset')})}) - self.assertEqual(gb.is_empty(), False) - - def test_is_empty_false_attribute(self): - """Test is_empty() when group has an attribute""" - gb = GroupBuilder('gb', attributes={'my_attr': 'attr_value'}) - self.assertEqual(gb.is_empty(), False) - - def test_is_empty_false_group_attribute(self): - """Test is_empty() when group has subgroup with an attribute""" - gb = GroupBuilder('gb', {'my_subgroup': GroupBuilder('my_subgroup', attributes={'my_attr': 'attr_value'})}) - self.assertEqual(gb.is_empty(), False) - - -class GroupBuilderDeepUpdateTests(unittest.TestCase): - - def test_mutually_exclusive_subgroups(self): - gb1 = GroupBuilder('gb1', {'subgroup1': GroupBuilder('subgroup1')}) - gb2 = GroupBuilder('gb2', {'subgroup2': GroupBuilder('subgroup2')}) - gb1.deep_update(gb2) - self.assertIn('subgroup2', gb1) - gb1sg = gb1['subgroup2'] - gb2sg = gb2['subgroup2'] - self.assertIs(gb1sg, gb2sg) - - def test_mutually_exclusive_datasets(self): - gb1 = GroupBuilder('gb1', datasets={'dataset1': DatasetBuilder('dataset1', [1, 2, 3])}) - gb2 = GroupBuilder('gb2', datasets={'dataset2': DatasetBuilder('dataset2', [4, 5, 6])}) - gb1.deep_update(gb2) - self.assertIn('dataset2', gb1) - # self.assertIs(gb1['dataset2'], gb2['dataset2']) - self.assertListEqual(gb1['dataset2'].data, gb2['dataset2'].data) - - def test_mutually_exclusive_attributes(self): - gb1 = GroupBuilder('gb1', attributes={'attr1': 'my_attribute1'}) - gb2 = GroupBuilder('gb2', attributes={'attr2': 'my_attribute2'}) - gb1.deep_update(gb2) - self.assertIn('attr2', gb2) - self.assertEqual(gb2['attr2'], 'my_attribute2') - - def test_mutually_exclusive_links(self): - gb1 = GroupBuilder('gb1', links={'link1': LinkBuilder(GroupBuilder('target1'), 'link1')}) - gb2 = GroupBuilder('gb2', links={'link2': LinkBuilder(GroupBuilder('target2'), 'link2')}) - gb1.deep_update(gb2) - self.assertIn('link2', gb2) - self.assertEqual(gb1['link2'], gb2['link2']) - - def test_intersecting_subgroups(self): - subgroup2 = GroupBuilder('subgroup2') - gb1 = GroupBuilder('gb1', {'subgroup1': GroupBuilder('subgroup1'), 'subgroup2': subgroup2}) - gb2 = GroupBuilder('gb2', {'subgroup2': GroupBuilder('subgroup2'), 'subgroup3': GroupBuilder('subgroup3')}) - gb1.deep_update(gb2) - self.assertIn('subgroup3', gb1) - self.assertIs(gb1['subgroup3'], gb2['subgroup3']) - self.assertIs(gb1['subgroup2'], subgroup2) - - def test_intersecting_datasets(self): - gb1 = GroupBuilder('gb1', datasets={'dataset2': DatasetBuilder('dataset2', [1, 2, 3])}) - gb2 = GroupBuilder('gb2', datasets={'dataset2': DatasetBuilder('dataset2', [4, 5, 6])}) - gb1.deep_update(gb2) - self.assertIn('dataset2', gb1) - self.assertListEqual(gb1['dataset2'].data, gb2['dataset2'].data) - - def test_intersecting_attributes(self): - gb1 = GroupBuilder('gb1', attributes={'attr2': 'my_attribute1'}) - gb2 = GroupBuilder('gb2', attributes={'attr2': 'my_attribute2'}) - gb1.deep_update(gb2) - self.assertIn('attr2', gb2) - self.assertEqual(gb2['attr2'], 'my_attribute2') - - def test_intersecting_links(self): - gb1 = GroupBuilder('gb1', links={'link2': LinkBuilder(GroupBuilder('target1'), 'link2')}) - gb2 = GroupBuilder('gb2', links={'link2': LinkBuilder(GroupBuilder('target2'), 'link2')}) - gb1.deep_update(gb2) - self.assertIn('link2', gb2) - self.assertEqual(gb1['link2'], gb2['link2']) - - -class DatasetBuilderDeepUpdateTests(unittest.TestCase): - - def test_overwrite(self): - db1 = DatasetBuilder('db1', [1, 2, 3]) - db2 = DatasetBuilder('db2', [4, 5, 6]) - db1.deep_update(db2) - self.assertListEqual(db1.data, db2.data) - - def test_no_overwrite(self): - db1 = DatasetBuilder('db1', [1, 2, 3]) - db2 = DatasetBuilder('db2', [4, 5, 6], attributes={'attr1': 'va1'}) - db1.deep_update(db2) - self.assertListEqual(db1.data, db2.data) - self.assertIn('attr1', db1.attributes) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/unit/form_tests/build_tests/test_io_manager.py b/tests/unit/form_tests/build_tests/test_io_manager.py deleted file mode 100644 index f94936527..000000000 --- a/tests/unit/form_tests/build_tests/test_io_manager.py +++ /dev/null @@ -1,318 +0,0 @@ -import unittest2 as unittest - -from pynwb.form import Container -from pynwb.form.spec import GroupSpec, AttributeSpec, DatasetSpec, SpecCatalog, SpecNamespace, NamespaceCatalog -from pynwb.form.spec.spec import ZERO_OR_MANY -from pynwb.form.build import GroupBuilder, DatasetBuilder -from pynwb.form.utils import docval, getargs -from pynwb.form.build import ObjectMapper, BuildManager, TypeMap - -from abc import ABCMeta -from six import with_metaclass - -CORE_NAMESPACE = 'test_core' - - -class Foo(Container): - - @docval({'name': 'name', 'type': str, 'doc': 'the name of this Foo'}, - {'name': 'my_data', 'type': list, 'doc': 'some data'}, - {'name': 'attr1', 'type': str, 'doc': 'an attribute'}, - {'name': 'attr2', 'type': int, 'doc': 'another attribute'}, - {'name': 'attr3', 'type': float, 'doc': 'a third attribute', 'default': 3.14}) - def __init__(self, **kwargs): - name, my_data, attr1, attr2, attr3 = getargs('name', 'my_data', 'attr1', 'attr2', 'attr3', kwargs) - super(Foo, self).__init__(name=name) - self.__data = my_data - self.__attr1 = attr1 - self.__attr2 = attr2 - self.__attr3 = attr3 - - def __eq__(self, other): - attrs = ('name', 'my_data', 'attr1', 'attr2', 'attr3') - return all(getattr(self, a) == getattr(other, a) for a in attrs) - - def __str__(self): - attrs = ('name', 'my_data', 'attr1', 'attr2', 'attr3') - return '<' + ','.join('%s=%s' % (a, getattr(self, a)) for a in attrs) + '>' - - @property - def my_data(self): - return self.__data - - @property - def attr1(self): - return self.__attr1 - - @property - def attr2(self): - return self.__attr2 - - @property - def attr3(self): - return self.__attr3 - - def __hash__(self): - return hash(self.name) - - -class FooBucket(Container): - - @docval({'name': 'name', 'type': str, 'doc': 'the name of this bucket'}, - {'name': 'foos', 'type': list, 'doc': 'the Foo objects in this bucket', 'default': list()}) - def __init__(self, **kwargs): - name, foos = getargs('name', 'foos', kwargs) - super(FooBucket, self).__init__(name=name) - self.__foos = foos - for f in self.__foos: - self.add_child(f) - - def __eq__(self, other): - return self.name == other.name and set(self.foos) == set(other.foos) - - def __str__(self): - foo_str = "[" + ",".join(str(f) for f in self.foos) + "]" - return 'name=%s, foos=%s' % (self.name, foo_str) - - @property - def foos(self): - return self.__foos - - -class TestBase(unittest.TestCase): - - def setUp(self): - self.foo_spec = GroupSpec('A test group specification with a data type', - data_type_def='Foo', - datasets=[DatasetSpec( - 'an example dataset', - 'int', - name='my_data', - attributes=[AttributeSpec( - 'attr2', - 'an example integer attribute', - 'int')])], - attributes=[AttributeSpec('attr1', 'an example string attribute', 'text')]) - - self.spec_catalog = SpecCatalog() - self.spec_catalog.register_spec(self.foo_spec, 'test.yaml') - self.namespace = SpecNamespace( - 'a test namespace', - CORE_NAMESPACE, - [{'source': 'test.yaml'}], - catalog=self.spec_catalog) - self.namespace_catalog = NamespaceCatalog() - self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace) - self.type_map = TypeMap(self.namespace_catalog) - self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo) - self.type_map.register_map(Foo, ObjectMapper) - self.manager = BuildManager(self.type_map) - - -class TestBuildManager(TestBase): - - def test_build(self): - container_inst = Foo('my_foo', list(range(10)), 'value1', 10) - expected = GroupBuilder( - 'my_foo', - datasets={ - 'my_data': - DatasetBuilder( - 'my_data', - list(range(10)), - attributes={'attr2': 10})}, - attributes={'attr1': 'value1', 'namespace': CORE_NAMESPACE, 'data_type': 'Foo'}) - builder1 = self.manager.build(container_inst) - self.assertDictEqual(builder1, expected) - - def test_build_memoization(self): - container_inst = Foo('my_foo', list(range(10)), 'value1', 10) - expected = GroupBuilder( - 'my_foo', - datasets={ - 'my_data': DatasetBuilder( - 'my_data', - list(range(10)), - attributes={'attr2': 10})}, - attributes={'attr1': 'value1', 'namespace': CORE_NAMESPACE, 'data_type': 'Foo'}) - builder1 = self.manager.build(container_inst) - builder2 = self.manager.build(container_inst) - self.assertDictEqual(builder1, expected) - self.assertIs(builder1, builder2) - - def test_construct(self): - builder = GroupBuilder( - 'my_foo', - datasets={ - 'my_data': DatasetBuilder( - 'my_data', - list(range(10)), - attributes={'attr2': 10})}, - attributes={'attr1': 'value1', 'namespace': CORE_NAMESPACE, 'data_type': 'Foo'}) - expected = Foo('my_foo', list(range(10)), 'value1', 10) # noqa: F841 - container = self.manager.construct(builder) - self.assertListEqual(container.my_data, list(range(10))) - self.assertEqual(container.attr1, 'value1') - self.assertEqual(container.attr2, 10) - - def test_construct_memoization(self): - builder = GroupBuilder( - 'my_foo', datasets={'my_data': DatasetBuilder( - 'my_data', - list(range(10)), - attributes={'attr2': 10})}, - attributes={'attr1': 'value1', 'namespace': CORE_NAMESPACE, 'data_type': 'Foo'}) - expected = Foo('my_foo', list(range(10)), 'value1', 10) # noqa: F841 - container1 = self.manager.construct(builder) - container2 = self.manager.construct(builder) - self.assertIs(container1, container2) - - -class TestNestedBase(with_metaclass(ABCMeta, TestBase)): - - def setUp(self): - super(TestNestedBase, self).setUp() - self.foo_bucket = FooBucket('test_foo_bucket', [ - Foo('my_foo1', list(range(10)), 'value1', 10), - Foo('my_foo2', list(range(10, 20)), 'value2', 20)]) - self.foo_builders = { - 'my_foo1': GroupBuilder('my_foo1', - datasets={'my_data': DatasetBuilder( - 'my_data', - list(range(10)), - attributes={'attr2': 10})}, - attributes={'attr1': 'value1', 'namespace': CORE_NAMESPACE, 'data_type': 'Foo'}), - 'my_foo2': GroupBuilder('my_foo2', datasets={'my_data': - DatasetBuilder( - 'my_data', - list(range(10, 20)), - attributes={'attr2': 20})}, - attributes={'attr1': 'value2', 'namespace': CORE_NAMESPACE, 'data_type': 'Foo'}) - } - self.setUpBucketBuilder() - self.setUpBucketSpec() - - self.spec_catalog.register_spec(self.bucket_spec, 'test.yaml') - self.type_map.register_container_type(CORE_NAMESPACE, 'FooBucket', FooBucket) - self.type_map.register_map(FooBucket, ObjectMapper) - self.manager = BuildManager(self.type_map) - - def setUpBucketBuilder(self): - raise unittest.SkipTest('Abstract Base Class') - - def setUpBucketSpec(self): - raise unittest.SkipTest('Abstract Base Class') - - def test_build(self): - ''' Test default mapping for an Container that has an Container as an attribute value ''' - builder = self.manager.build(self.foo_bucket) - self.assertDictEqual(builder, self.bucket_builder) - - def test_construct(self): - container = self.manager.construct(self.bucket_builder) - self.assertEqual(container, self.foo_bucket) - - -class TestNestedContainersNoSubgroups(TestNestedBase): - ''' - Test BuildManager.build and BuildManager.construct when the - Container contains other Containers, but does not keep them in - additional subgroups - ''' - - def setUpBucketBuilder(self): - self.bucket_builder = GroupBuilder( - 'test_foo_bucket', - groups=self.foo_builders, - attributes={'namespace': CORE_NAMESPACE, 'data_type': 'FooBucket'}) - - def setUpBucketSpec(self): - self.bucket_spec = GroupSpec('A test group specification for a data type containing data type', - name="test_foo_bucket", - data_type_def='FooBucket', - groups=[GroupSpec( - 'the Foos in this bucket', - data_type_inc='Foo', - quantity=ZERO_OR_MANY)]) - - -class TestNestedContainersSubgroup(TestNestedBase): - ''' - Test BuildManager.build and BuildManager.construct when the - Container contains other Containers that are stored in a subgroup - ''' - - def setUpBucketBuilder(self): - tmp_builder = GroupBuilder('foo_holder', groups=self.foo_builders) - self.bucket_builder = GroupBuilder( - 'test_foo_bucket', - groups={'foos': tmp_builder}, - attributes={'namespace': - CORE_NAMESPACE, 'data_type': 'FooBucket'}) - - def setUpBucketSpec(self): - tmp_spec = GroupSpec( - 'A subgroup for Foos', - name='foo_holder', - groups=[GroupSpec('the Foos in this bucket', - data_type_inc='Foo', - quantity=ZERO_OR_MANY)]) - self.bucket_spec = GroupSpec('A test group specification for a data type containing data type', - name="test_foo_bucket", - data_type_def='FooBucket', - groups=[tmp_spec]) - - -class TestNestedContainersSubgroupSubgroup(TestNestedBase): - ''' - Test BuildManager.build and BuildManager.construct when the - Container contains other Containers that are stored in a subgroup - in a subgroup - ''' - - def setUpBucketBuilder(self): - tmp_builder = GroupBuilder('foo_holder', groups=self.foo_builders) - tmp_builder = GroupBuilder('foo_holder_holder', groups={'foo_holder': tmp_builder}) - self.bucket_builder = GroupBuilder( - 'test_foo_bucket', - groups={'foo_holder': tmp_builder}, - attributes={'namespace': CORE_NAMESPACE, - 'data_type': 'FooBucket'}) - - def setUpBucketSpec(self): - tmp_spec = GroupSpec('A subgroup for Foos', - name='foo_holder', - groups=[GroupSpec('the Foos in this bucket', - data_type_inc='Foo', - quantity=ZERO_OR_MANY)]) - tmp_spec = GroupSpec('A subgroup to hold the subgroup', name='foo_holder_holder', groups=[tmp_spec]) - self.bucket_spec = GroupSpec('A test group specification for a data type containing data type', - name="test_foo_bucket", - data_type_def='FooBucket', - groups=[tmp_spec]) - - -class TestTypeMap(TestBase): - - def test_get_ns_dt_missing(self): - bldr = GroupBuilder('my_foo', attributes={'attr1': 'value1'}) - dt = self.type_map.get_builder_dt(bldr) - ns = self.type_map.get_builder_ns(bldr) - self.assertIsNone(dt) - self.assertIsNone(ns) - - def test_get_ns_dt(self): - bldr = GroupBuilder('my_foo', attributes={'attr1': 'value1', 'namespace': 'CORE', 'data_type': 'Foo'}) - dt = self.type_map.get_builder_dt(bldr) - ns = self.type_map.get_builder_ns(bldr) - self.assertEqual(dt, 'Foo') - self.assertEqual(ns, 'CORE') - - -# TODO: -class TestWildCardNamedSpecs(unittest.TestCase): - pass - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/unit/form_tests/build_tests/test_io_map.py b/tests/unit/form_tests/build_tests/test_io_map.py deleted file mode 100644 index 69e585482..000000000 --- a/tests/unit/form_tests/build_tests/test_io_map.py +++ /dev/null @@ -1,318 +0,0 @@ -import unittest2 as unittest - -from pynwb.form.spec import GroupSpec, AttributeSpec, DatasetSpec, SpecCatalog, SpecNamespace, NamespaceCatalog -from pynwb.form.build import GroupBuilder, DatasetBuilder, ObjectMapper, BuildManager, TypeMap -from pynwb.form import Container -from pynwb.form.utils import docval, getargs, get_docval - -from abc import ABCMeta -from six import with_metaclass - -CORE_NAMESPACE = 'test_core' - - -class Bar(Container): - - @docval({'name': 'name', 'type': str, 'doc': 'the name of this Bar'}, - {'name': 'data', 'type': list, 'doc': 'some data'}, - {'name': 'attr1', 'type': str, 'doc': 'an attribute'}, - {'name': 'attr2', 'type': int, 'doc': 'another attribute'}, - {'name': 'attr3', 'type': float, 'doc': 'a third attribute', 'default': 3.14}) - def __init__(self, **kwargs): - name, data, attr1, attr2, attr3 = getargs('name', 'data', 'attr1', 'attr2', 'attr3', kwargs) - super(Bar, self).__init__(name=name) - self.__data = data - self.__attr1 = attr1 - self.__attr2 = attr2 - self.__attr3 = attr3 - - def __eq__(self, other): - attrs = ('name', 'data', 'attr1', 'attr2', 'attr3') - return all(getattr(self, a) == getattr(other, a) for a in attrs) - - def __str__(self): - attrs = ('name', 'data', 'attr1', 'attr2', 'attr3') - return ','.join('%s=%s' % (a, getattr(self, a)) for a in attrs) - - @property - def data_type(self): - return 'Bar' - - @property - def data(self): - return self.__data - - @property - def attr1(self): - return self.__attr1 - - @property - def attr2(self): - return self.__attr2 - - @property - def attr3(self): - return self.__attr3 - - -class Foo(Container): - pass - - -class TestGetSubSpec(unittest.TestCase): - - def setUp(self): - self.bar_spec = GroupSpec('A test group specification with a data type', data_type_def='Bar') - spec_catalog = SpecCatalog() - spec_catalog.register_spec(self.bar_spec, 'test.yaml') - namespace = SpecNamespace('a test namespace', CORE_NAMESPACE, [{'source': 'test.yaml'}], catalog=spec_catalog) - namespace_catalog = NamespaceCatalog() - namespace_catalog.add_namespace(CORE_NAMESPACE, namespace) - self.type_map = TypeMap(namespace_catalog) - self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar) - - def test_get_subspec_data_type_noname(self): - parent_spec = GroupSpec('Something to hold a Bar', 'bar_bucket', groups=[self.bar_spec]) - sub_builder = GroupBuilder('my_bar', attributes={'data_type': 'Bar', 'namespace': CORE_NAMESPACE}) - builder = GroupBuilder('bar_bucket', groups={'my_bar': sub_builder}) # noqa: F841 - result = self.type_map.get_subspec(parent_spec, sub_builder) - self.assertIs(result, self.bar_spec) - - def test_get_subspec_named(self): - child_spec = GroupSpec('A test group specification with a data type', 'my_subgroup') - parent_spec = GroupSpec('Something to hold a Bar', 'my_group', groups=[child_spec]) - sub_builder = GroupBuilder('my_subgroup', attributes={'data_type': 'Bar', 'namespace': CORE_NAMESPACE}) - builder = GroupBuilder('my_group', groups={'my_bar': sub_builder}) # noqa: F841 - result = self.type_map.get_subspec(parent_spec, sub_builder) - self.assertIs(result, child_spec) - - -class TestTypeMap(unittest.TestCase): - - def setUp(self): - self.bar_spec = GroupSpec('A test group specification with a data type', data_type_def='Bar') - self.foo_spec = GroupSpec('A test group specification with data type Foo', data_type_def='Foo') - self.spec_catalog = SpecCatalog() - self.spec_catalog.register_spec(self.bar_spec, 'test.yaml') - self.spec_catalog.register_spec(self.foo_spec, 'test.yaml') - self.namespace = SpecNamespace('a test namespace', CORE_NAMESPACE, [{'source': 'test.yaml'}], - catalog=self.spec_catalog) - self.namespace_catalog = NamespaceCatalog() - self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace) - self.type_map = TypeMap(self.namespace_catalog) - self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar) - self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo) - # self.build_manager = BuildManager(self.type_map) - - def test_get_map_unique_mappers(self): - self.type_map.register_map(Bar, ObjectMapper) - self.type_map.register_map(Foo, ObjectMapper) - bar_inst = Bar('my_bar', list(range(10)), 'value1', 10) - foo_inst = Foo(name='my_foo') - bar_mapper = self.type_map.get_map(bar_inst) - foo_mapper = self.type_map.get_map(foo_inst) - self.assertIsNot(bar_mapper, foo_mapper) - - def test_get_map(self): - self.type_map.register_map(Bar, ObjectMapper) - container_inst = Bar('my_bar', list(range(10)), 'value1', 10) - mapper = self.type_map.get_map(container_inst) - self.assertIsInstance(mapper, ObjectMapper) - self.assertIs(mapper.spec, self.bar_spec) - mapper2 = self.type_map.get_map(container_inst) - self.assertIs(mapper, mapper2) - - def test_get_map_register(self): - class MyMap(ObjectMapper): - pass - self.type_map.register_map(Bar, MyMap) - - container_inst = Bar('my_bar', list(range(10)), 'value1', 10) - mapper = self.type_map.get_map(container_inst) - self.assertIs(mapper.spec, self.bar_spec) - self.assertIsInstance(mapper, MyMap) - - -class TestDynamicContainer(unittest.TestCase): - - def setUp(self): - self.bar_spec = GroupSpec('A test group specification with a data type', - data_type_def='Bar', - datasets=[DatasetSpec('an example dataset', 'int', name='data', - attributes=[AttributeSpec( - 'attr2', 'an example integer attribute', 'int')])], - attributes=[AttributeSpec('attr1', 'an example string attribute', 'text')]) - self.spec_catalog = SpecCatalog() - self.spec_catalog.register_spec(self.bar_spec, 'test.yaml') - self.namespace = SpecNamespace('a test namespace', CORE_NAMESPACE, - [{'source': 'test.yaml'}], catalog=self.spec_catalog) - self.namespace_catalog = NamespaceCatalog() - self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace) - self.type_map = TypeMap(self.namespace_catalog) - self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar) - self.type_map.register_map(Bar, ObjectMapper) - self.manager = BuildManager(self.type_map) - self.mapper = ObjectMapper(self.bar_spec) - - def test_dynamic_container_creation(self): - baz_spec = GroupSpec('A test extension with no Container class', - data_type_def='Baz', data_type_inc=self.bar_spec, - attributes=[AttributeSpec('attr3', 'an example float attribute', 'float'), - AttributeSpec('attr4', 'another example float attribute', 'float')]) - self.spec_catalog.register_spec(baz_spec, 'extension.yaml') - cls = self.type_map.get_container_cls(CORE_NAMESPACE, 'Baz') - expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4'} - received_args = set() - for x in get_docval(cls.__init__): - received_args.add(x['name']) - with self.subTest(name=x['name']): - self.assertNotIn('default', x) - self.assertSetEqual(expected_args, received_args) - self.assertEqual(cls.__name__, 'Baz') - self.assertTrue(issubclass(cls, Bar)) - - def test_dynamic_container_creation_defaults(self): - baz_spec = GroupSpec('A test extension with no Container class', - data_type_def='Baz', data_type_inc=self.bar_spec, - attributes=[AttributeSpec('attr3', 'an example float attribute', 'float'), - AttributeSpec('attr4', 'another example float attribute', 'float')]) - self.spec_catalog.register_spec(baz_spec, 'extension.yaml') - cls = self.type_map.get_container_cls(CORE_NAMESPACE, 'Baz') - expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4'} - received_args = set(map(lambda x: x['name'], get_docval(cls.__init__))) - self.assertSetEqual(expected_args, received_args) - self.assertEqual(cls.__name__, 'Baz') - self.assertTrue(issubclass(cls, Bar)) - - def test_dynamic_container_constructor(self): - baz_spec = GroupSpec('A test extension with no Container class', - data_type_def='Baz', data_type_inc=self.bar_spec, - attributes=[AttributeSpec('attr3', 'an example float attribute', 'float'), - AttributeSpec('attr4', 'another example float attribute', 'float')]) - self.spec_catalog.register_spec(baz_spec, 'extension.yaml') - cls = self.type_map.get_container_cls(CORE_NAMESPACE, 'Baz') - # TODO: test that constructor works! - inst = cls('My Baz', [1, 2, 3, 4], 'string attribute', 1000, attr3=98.6, attr4=1.0) - self.assertEqual(inst.name, 'My Baz') - self.assertEqual(inst.data, [1, 2, 3, 4]) - self.assertEqual(inst.attr1, 'string attribute') - self.assertEqual(inst.attr2, 1000) - self.assertEqual(inst.attr3, 98.6) - self.assertEqual(inst.attr4, 1.0) - - -class TestObjectMapper(with_metaclass(ABCMeta, unittest.TestCase)): - - def setUp(self): - self.setUpBarSpec() - self.spec_catalog = SpecCatalog() - self.spec_catalog.register_spec(self.bar_spec, 'test.yaml') - self.namespace = SpecNamespace('a test namespace', CORE_NAMESPACE, - [{'source': 'test.yaml'}], catalog=self.spec_catalog) - self.namespace_catalog = NamespaceCatalog() - self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace) - self.type_map = TypeMap(self.namespace_catalog) - self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar) - self.type_map.register_map(Bar, ObjectMapper) - self.manager = BuildManager(self.type_map) - self.mapper = ObjectMapper(self.bar_spec) - - def setUpBarSpec(self): - raise unittest.SkipTest('setUpBarSpec not implemented') - - def test_default_mapping(self): - attr_map = self.mapper.get_attr_names(self.bar_spec) - keys = set(attr_map.keys()) - for key in keys: - with self.subTest(key=key): - self.assertIs(attr_map[key], self.mapper.get_attr_spec(key)) - self.assertIs(attr_map[key], self.mapper.get_carg_spec(key)) - - -class TestObjectMapperNested(TestObjectMapper): - - def setUpBarSpec(self): - self.bar_spec = GroupSpec('A test group specification with a data type', - data_type_def='Bar', - datasets=[DatasetSpec('an example dataset', 'int', name='data', - attributes=[AttributeSpec( - 'attr2', 'an example integer attribute', 'int')])], - attributes=[AttributeSpec('attr1', 'an example string attribute', 'text')]) - - def test_build(self): - ''' Test default mapping functionality when object attributes map to an attribute deeper - than top-level Builder ''' - container_inst = Bar('my_bar', list(range(10)), 'value1', 10) - expected = GroupBuilder('my_bar', datasets={'data': DatasetBuilder( - 'data', list(range(10)), attributes={'attr2': 10})}, - attributes={'attr1': 'value1'}) - builder = self.mapper.build(container_inst, self.manager) - self.assertDictEqual(builder, expected) - - def test_construct(self): - ''' Test default mapping functionality when object attributes map to an attribute - deeper than top-level Builder ''' - builder = GroupBuilder('my_bar', datasets={'data': DatasetBuilder( - 'data', list(range(10)), attributes={'attr2': 10})}, - attributes={'attr1': 'value1', 'data_type': 'Bar', 'namespace': CORE_NAMESPACE}) - expected = Bar('my_bar', list(range(10)), 'value1', 10) - container = self.mapper.construct(builder, self.manager) - self.assertEqual(container, expected) - - def test_default_mapping_keys(self): - attr_map = self.mapper.get_attr_names(self.bar_spec) - keys = set(attr_map.keys()) - expected = {'attr1', 'data', 'attr2'} - self.assertSetEqual(keys, expected) - - -class TestObjectMapperNoNesting(TestObjectMapper): - - def setUpBarSpec(self): - self.bar_spec = GroupSpec('A test group specification with a data type', - data_type_def='Bar', - datasets=[DatasetSpec('an example dataset', 'int', name='data')], - attributes=[AttributeSpec('attr1', 'an example string attribute', 'text'), - AttributeSpec('attr2', 'an example integer attribute', 'int')]) - - def test_build(self): - ''' Test default mapping functionality when no attributes are nested ''' - container = Bar('my_bar', list(range(10)), 'value1', 10) - builder = self.mapper.build(container, self.manager) - expected = GroupBuilder('my_bar', datasets={'data': DatasetBuilder('data', list(range(10)))}, - attributes={'attr1': 'value1', 'attr2': 10}) - self.assertDictEqual(builder, expected) - - def test_construct(self): - builder = GroupBuilder('my_bar', datasets={'data': DatasetBuilder('data', list(range(10)))}, - attributes={'attr1': 'value1', 'attr2': 10, 'data_type': 'Bar', - 'namespace': CORE_NAMESPACE}) - expected = Bar('my_bar', list(range(10)), 'value1', 10) - container = self.mapper.construct(builder, self.manager) - self.assertEqual(container, expected) - - def test_default_mapping_keys(self): - attr_map = self.mapper.get_attr_names(self.bar_spec) - keys = set(attr_map.keys()) - expected = {'attr1', 'data', 'attr2'} - self.assertSetEqual(keys, expected) - - -class TestObjectMapperContainer(TestObjectMapper): - - def setUpBarSpec(self): - self.bar_spec = GroupSpec('A test group specification with a data type', - data_type_def='Bar', - groups=[GroupSpec('an example group', data_type_def='Foo')], - attributes=[AttributeSpec('attr1', 'an example string attribute', 'text'), - AttributeSpec('attr2', 'an example integer attribute', 'int')]) - - def test_default_mapping_keys(self): - attr_map = self.mapper.get_attr_names(self.bar_spec) - keys = set(attr_map.keys()) - expected = {'attr1', 'foo', 'attr2'} - self.assertSetEqual(keys, expected) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/unit/form_tests/build_tests/test_io_map_data.py b/tests/unit/form_tests/build_tests/test_io_map_data.py deleted file mode 100644 index 90b674534..000000000 --- a/tests/unit/form_tests/build_tests/test_io_map_data.py +++ /dev/null @@ -1,56 +0,0 @@ -import unittest2 as unittest - -from pynwb.form.spec import AttributeSpec, DatasetSpec, SpecCatalog, SpecNamespace, NamespaceCatalog -from pynwb.form.build import DatasetBuilder, ObjectMapper, BuildManager, TypeMap -from pynwb.form import Data -from pynwb.form.utils import docval, getargs - -CORE_NAMESPACE = 'test_core' - - -class Baz(Data): - - @docval({'name': 'name', 'type': str, 'doc': 'the name of this Baz'}, - {'name': 'data', 'type': list, 'doc': 'some data'}, - {'name': 'baz_attr', 'type': str, 'doc': 'an attribute'}) - def __init__(self, **kwargs): - name, data, baz_attr = getargs('name', 'data', 'baz_attr', kwargs) - super(Baz, self).__init__(name=name) - self.__data = data - self.__baz_attr = baz_attr - - @property - def data(self): - return self.__data - - @property - def baz_attr(self): - return self.__baz_attr - - -class TestDataMap(unittest.TestCase): - - def setUp(self): - self.setUpBazSpec() - self.spec_catalog = SpecCatalog() - self.spec_catalog.register_spec(self.baz_spec, 'test.yaml') - self.namespace = SpecNamespace('a test namespace', CORE_NAMESPACE, [{'source': 'test.yaml'}], - catalog=self.spec_catalog) - self.namespace_catalog = NamespaceCatalog() - self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace) - self.type_map = TypeMap(self.namespace_catalog) - self.type_map.register_container_type(CORE_NAMESPACE, 'Baz', Baz) - self.type_map.register_map(Baz, ObjectMapper) - self.manager = BuildManager(self.type_map) - self.mapper = ObjectMapper(self.baz_spec) - - def setUpBazSpec(self): - self.baz_spec = DatasetSpec('an Baz type', 'int', name='MyBaz', data_type_def='Baz', - attributes=[AttributeSpec('baz_attr', 'an example string attribute', 'text')]) - - def test_build(self): - ''' Test default mapping functionality when no attributes are nested ''' - container = Baz('my_baz', list(range(10)), 'abcdefghijklmnopqrstuvwxyz') - builder = self.mapper.build(container, self.manager) - expected = DatasetBuilder('my_baz', list(range(10)), attributes={'baz_attr': 'abcdefghijklmnopqrstuvwxyz'}) - self.assertDictEqual(builder, expected) diff --git a/tests/unit/form_tests/spec_tests/__init__.py b/tests/unit/form_tests/spec_tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/unit/form_tests/spec_tests/test_attribute_spec.py b/tests/unit/form_tests/spec_tests/test_attribute_spec.py deleted file mode 100644 index d91057fd1..000000000 --- a/tests/unit/form_tests/spec_tests/test_attribute_spec.py +++ /dev/null @@ -1,66 +0,0 @@ -import unittest2 as unittest -import json - -from pynwb.form.spec import AttributeSpec - - -class AttributeSpecTests(unittest.TestCase): - - def test_constructor(self): - spec = AttributeSpec('attribute1', - 'my first attribute', - 'text') - self.assertEqual(spec['name'], 'attribute1') - self.assertEqual(spec['dtype'], 'text') - self.assertEqual(spec['doc'], 'my first attribute') - self.assertIsNone(spec.parent) - json.dumps(spec) # to ensure there are no circular links - - def test_invalid_dtype(self): - with self.assertRaises(ValueError): - AttributeSpec(name='attribute1', - doc='my first attribute', - dtype='invalid' # <-- Invalid dtype must raise a ValueError - ) - - def test_both_value_and_default_value_set(self): - with self.assertRaises(ValueError): - AttributeSpec(name='attribute1', - doc='my first attribute', - dtype='int', - value=5, - default_value=10 # <-- Default_value and value can't be set at the same time - ) - - def test_colliding_shape_and_dims(self): - with self.assertRaises(ValueError): - AttributeSpec(name='attribute1', - doc='my first attribute', - dtype='int', - dims=['test'], - shape=[None, 2] # <-- Length of shape and dims do not match must raise a ValueError - ) - - def test_default_value(self): - spec = AttributeSpec('attribute1', - 'my first attribute', - 'text', - default_value='some text') - self.assertEqual(spec['default_value'], 'some text') - self.assertEqual(spec.default_value, 'some text') - - def test_shape(self): - shape = [None, 2] - spec = AttributeSpec('attribute1', - 'my first attribute', - 'text', - shape=shape) - self.assertEqual(spec['shape'], shape) - self.assertEqual(spec.shape, shape) - - def test_dims_without_shape(self): - spec = AttributeSpec('attribute1', - 'my first attribute', - 'text', - dims=['test']) - self.assertEqual(spec.shape, (None, )) diff --git a/tests/unit/form_tests/spec_tests/test_dataset_spec.py b/tests/unit/form_tests/spec_tests/test_dataset_spec.py deleted file mode 100644 index dfdbd9c1b..000000000 --- a/tests/unit/form_tests/spec_tests/test_dataset_spec.py +++ /dev/null @@ -1,232 +0,0 @@ -import unittest2 as unittest -import json - -from pynwb.form.spec import GroupSpec, DatasetSpec, AttributeSpec, DtypeSpec, RefSpec - - -class DatasetSpecTests(unittest.TestCase): - def setUp(self): - self.attributes = [ - AttributeSpec('attribute1', 'my first attribute', 'text'), - AttributeSpec('attribute2', 'my second attribute', 'text') - ] - - def test_constructor(self): - spec = DatasetSpec('my first dataset', - 'int', - name='dataset1', - attributes=self.attributes) - self.assertEqual(spec['dtype'], 'int') - self.assertEqual(spec['name'], 'dataset1') - self.assertEqual(spec['doc'], 'my first dataset') - self.assertNotIn('linkable', spec) - self.assertNotIn('data_type_def', spec) - self.assertListEqual(spec['attributes'], self.attributes) - self.assertIs(spec, self.attributes[0].parent) - self.assertIs(spec, self.attributes[1].parent) - json.dumps(spec) - - def test_constructor_nwbtype(self): - spec = DatasetSpec('my first dataset', - 'int', - name='dataset1', - attributes=self.attributes, - linkable=False, - data_type_def='EphysData') - self.assertEqual(spec['dtype'], 'int') - self.assertEqual(spec['name'], 'dataset1') - self.assertEqual(spec['doc'], 'my first dataset') - self.assertEqual(spec['data_type_def'], 'EphysData') - self.assertFalse(spec['linkable']) - self.assertListEqual(spec['attributes'], self.attributes) - self.assertIs(spec, self.attributes[0].parent) - self.assertIs(spec, self.attributes[1].parent) - - def test_constructor_shape(self): - shape = [None, 2] - spec = DatasetSpec('my first dataset', - 'int', - name='dataset1', - shape=shape, - attributes=self.attributes) - self.assertEqual(spec['shape'], shape) - self.assertEqual(spec.shape, shape) - - def test_constructor_invalidate_dtype(self): - with self.assertRaises(ValueError): - DatasetSpec(doc='my first dataset', - dtype='my bad dtype', # <-- Expect AssertionError due to bad type - name='dataset1', - dims=(None, None), - attributes=self.attributes, - linkable=False, - data_type_def='EphysData') - - def test_constructor_ref_spec(self): - dtype = RefSpec('TimeSeries', 'object') - spec = DatasetSpec(doc='my first dataset', - dtype=dtype, - name='dataset1', - dims=(None, None), - attributes=self.attributes, - linkable=False, - data_type_def='EphysData') - self.assertDictEqual(spec['dtype'], dtype) - - def test_datatype_extension(self): - base = DatasetSpec('my first dataset', - 'int', - name='dataset1', - attributes=self.attributes, - linkable=False, - data_type_def='EphysData') - - attributes = [AttributeSpec('attribute3', 'my first extending attribute', 'float')] - ext = DatasetSpec('my first dataset extension', - 'int', - name='dataset1', - attributes=attributes, - linkable=False, - data_type_inc=base, - data_type_def='SpikeData') - self.assertDictEqual(ext['attributes'][0], attributes[0]) - self.assertDictEqual(ext['attributes'][1], self.attributes[0]) - self.assertDictEqual(ext['attributes'][2], self.attributes[1]) - ext_attrs = ext.attributes - self.assertIs(ext, ext_attrs[0].parent) - self.assertIs(ext, ext_attrs[1].parent) - self.assertIs(ext, ext_attrs[2].parent) - - def test_datatype_extension_groupspec(self): - '''Test to make sure DatasetSpec catches when a GroupSpec used as data_type_inc''' - base = GroupSpec('a fake grop', - data_type_def='EphysData') - with self.assertRaises(TypeError): - ext = DatasetSpec('my first dataset extension', # noqa: F841 - 'int', - name='dataset1', - data_type_inc=base, - data_type_def='SpikeData') - - def test_constructor_table(self): - dtype1 = DtypeSpec('column1', 'the first column', 'int') - dtype2 = DtypeSpec('column2', 'the second column', 'float') - spec = DatasetSpec('my first table', - [dtype1, dtype2], - name='table1', - attributes=self.attributes) - self.assertEqual(spec['dtype'], [dtype1, dtype2]) - self.assertEqual(spec['name'], 'table1') - self.assertEqual(spec['doc'], 'my first table') - self.assertNotIn('linkable', spec) - self.assertNotIn('data_type_def', spec) - self.assertListEqual(spec['attributes'], self.attributes) - self.assertIs(spec, self.attributes[0].parent) - self.assertIs(spec, self.attributes[1].parent) - json.dumps(spec) - - def test_constructor_invalid_table(self): - with self.assertRaises(ValueError): - DatasetSpec('my first table', - [DtypeSpec('column1', 'the first column', 'int'), - {} # <--- Bad compound type spec must raise an error - ], - name='table1', - attributes=self.attributes) - - def test_constructor_default_value(self): - spec = DatasetSpec(doc='test', - default_value=5, - dtype='int', - data_type_def='test') - self.assertEqual(spec.default_value, 5) - - def test_name_with_incompatible_quantity(self): - # Check that we raise an error when the quantity allows more than one instance with a fixed name - with self.assertRaises(ValueError): - DatasetSpec(doc='my first dataset', - dtype='int', - name='ds1', - quantity='zero_or_many') - with self.assertRaises(ValueError): - DatasetSpec(doc='my first dataset', - dtype='int', - name='ds1', - quantity='one_or_many') - - def test_name_with_compatible_quantity(self): - # Make sure compatible quantity flags pass when name is fixed - DatasetSpec(doc='my first dataset', - dtype='int', - name='ds1', - quantity='zero_or_one') - DatasetSpec(doc='my first dataset', - dtype='int', - name='ds1', - quantity=1) - - def test_datatype_table_extension(self): - dtype1 = DtypeSpec('column1', 'the first column', 'int') - dtype2 = DtypeSpec('column2', 'the second column', 'float') - base = DatasetSpec('my first table', - [dtype1, dtype2], - attributes=self.attributes, - data_type_def='SimpleTable') - self.assertEqual(base['dtype'], [dtype1, dtype2]) - self.assertEqual(base['doc'], 'my first table') - dtype3 = DtypeSpec('column3', 'the third column', 'text') - ext = DatasetSpec('my first table extension', - [dtype3], - data_type_inc=base, - data_type_def='ExtendedTable') - self.assertEqual(ext['dtype'], [dtype1, dtype2, dtype3]) - self.assertEqual(ext['doc'], 'my first table extension') - - def test_datatype_table_extension_higher_precision(self): - dtype1 = DtypeSpec('column1', 'the first column', 'int') - dtype2 = DtypeSpec('column2', 'the second column', 'float32') - base = DatasetSpec('my first table', - [dtype1, dtype2], - attributes=self.attributes, - data_type_def='SimpleTable') - self.assertEqual(base['dtype'], [dtype1, dtype2]) - self.assertEqual(base['doc'], 'my first table') - dtype3 = DtypeSpec('column2', 'the second column, with greater precision', 'float64') - ext = DatasetSpec('my first table extension', - [dtype3], - data_type_inc=base, - data_type_def='ExtendedTable') - self.assertEqual(ext['dtype'], [dtype1, dtype3]) - self.assertEqual(ext['doc'], 'my first table extension') - - def test_datatype_table_extension_lower_precision(self): - dtype1 = DtypeSpec('column1', 'the first column', 'int') - dtype2 = DtypeSpec('column2', 'the second column', 'float64') - base = DatasetSpec('my first table', - [dtype1, dtype2], - attributes=self.attributes, - data_type_def='SimpleTable') - self.assertEqual(base['dtype'], [dtype1, dtype2]) - self.assertEqual(base['doc'], 'my first table') - dtype3 = DtypeSpec('column2', 'the second column, with greater precision', 'float32') - with self.assertRaisesRegex(ValueError, 'Cannot extend float64 to float32'): - ext = DatasetSpec('my first table extension', # noqa: F841 - [dtype3], - data_type_inc=base, - data_type_def='ExtendedTable') - - def test_datatype_table_extension_diff_format(self): - dtype1 = DtypeSpec('column1', 'the first column', 'int') - dtype2 = DtypeSpec('column2', 'the second column', 'float64') - base = DatasetSpec('my first table', - [dtype1, dtype2], - attributes=self.attributes, - data_type_def='SimpleTable') - self.assertEqual(base['dtype'], [dtype1, dtype2]) - self.assertEqual(base['doc'], 'my first table') - dtype3 = DtypeSpec('column2', 'the second column, with greater precision', 'int32') - with self.assertRaisesRegex(ValueError, 'Cannot extend float64 to int32'): - ext = DatasetSpec('my first table extension', # noqa: F841 - [dtype3], - data_type_inc=base, - data_type_def='ExtendedTable') diff --git a/tests/unit/form_tests/spec_tests/test_dtype_spec.py b/tests/unit/form_tests/spec_tests/test_dtype_spec.py deleted file mode 100644 index 2ab806bc2..000000000 --- a/tests/unit/form_tests/spec_tests/test_dtype_spec.py +++ /dev/null @@ -1,62 +0,0 @@ -import unittest2 as unittest - -from pynwb.form.spec import DtypeSpec, DtypeHelper, RefSpec - - -class DtypeSpecHelper(unittest.TestCase): - def setUp(self): - pass - - def test_recommended_dtypes(self): - self.assertListEqual(DtypeHelper.recommended_primary_dtypes, - list(DtypeHelper.primary_dtype_synonyms.keys())) - - def test_valid_primary_dtypes(self): - a = set(list(DtypeHelper.primary_dtype_synonyms.keys()) + - [vi for v in DtypeHelper.primary_dtype_synonyms.values() for vi in v]) - self.assertSetEqual(a, DtypeHelper.valid_primary_dtypes) - - def test_simplify_cpd_type(self): - compound_type = [DtypeSpec('test', 'test field', 'float'), - DtypeSpec('test2', 'test field2', 'int')] - expected_result = ['float', 'int'] - result = DtypeHelper.simplify_cpd_type(compound_type) - self.assertListEqual(result, expected_result) - - -class DtypeSpecTests(unittest.TestCase): - def setUp(self): - pass - - def test_constructor(self): - spec = DtypeSpec('column1', 'an example column', 'int') - self.assertEqual(spec.doc, 'an example column') - self.assertEqual(spec.name, 'column1') - self.assertEqual(spec.dtype, 'int') - - def test_build_spec(self): - spec = DtypeSpec.build_spec({'doc': 'an example column', 'name': 'column1', 'dtype': 'int'}) - self.assertEqual(spec.doc, 'an example column') - self.assertEqual(spec.name, 'column1') - self.assertEqual(spec.dtype, 'int') - - def test_invalid_refspec_dict(self): - with self.assertRaises(AssertionError): - DtypeSpec.assertValidDtype({'no target': 'test', # <-- missing or here bad target key for RefSpec - 'reftype': 'object'}) - - def test_refspec_dtype(self): - # just making sure this does not cause an error - DtypeSpec('column1', 'an example column', RefSpec('TimeSeries', 'object')) - - def test_invalid_dtype(self): - with self.assertRaises(AssertionError): - DtypeSpec('column1', 'an example column', - dtype='bad dtype' # <-- make sure a bad type string raises an error - ) - - def test_is_ref(self): - spec = DtypeSpec('column1', 'an example column', RefSpec('TimeSeries', 'object')) - self.assertTrue(DtypeSpec.is_ref(spec)) - spec = DtypeSpec('column1', 'an example column', 'int') - self.assertFalse(DtypeSpec.is_ref(spec)) diff --git a/tests/unit/form_tests/spec_tests/test_group_spec.py b/tests/unit/form_tests/spec_tests/test_group_spec.py deleted file mode 100644 index f09f84da5..000000000 --- a/tests/unit/form_tests/spec_tests/test_group_spec.py +++ /dev/null @@ -1,211 +0,0 @@ -import unittest2 as unittest -import json - -from pynwb.form.spec import GroupSpec, DatasetSpec, AttributeSpec - - -class GroupSpecTests(unittest.TestCase): - def setUp(self): - self.attributes = [ - AttributeSpec('attribute1', 'my first attribute', 'text'), - AttributeSpec('attribute2', 'my second attribute', 'text') - ] - - self.dset1_attributes = [ - AttributeSpec('attribute3', 'my third attribute', 'text'), - AttributeSpec('attribute4', 'my fourth attribute', 'text') - ] - - self.dset2_attributes = [ - AttributeSpec('attribute5', 'my fifth attribute', 'text'), - AttributeSpec('attribute6', 'my sixth attribute', 'text') - ] - - self.datasets = [ - DatasetSpec('my first dataset', - 'int', - name='dataset1', - attributes=self.dset1_attributes, - linkable=True), - DatasetSpec('my second dataset', - 'int', - name='dataset2', - attributes=self.dset2_attributes, - linkable=True, - data_type_def='VoltageArray') - ] - - self.subgroups = [ - GroupSpec('A test subgroup', - name='subgroup1', - linkable=False), - GroupSpec('A test subgroup', - name='subgroup2', - linkable=False) - - ] - self.ndt_attr_spec = AttributeSpec('data_type', 'the data type of this object', 'text', value='EphysData') - self.ns_attr_spec = AttributeSpec('namespace', 'the namespace for the data type of this object', - 'text', required=False) - - def test_constructor(self): - spec = GroupSpec('A test group', - name='root_constructor', - groups=self.subgroups, - datasets=self.datasets, - attributes=self.attributes, - linkable=False) - self.assertFalse(spec['linkable']) - self.assertListEqual(spec['attributes'], self.attributes) - self.assertListEqual(spec['datasets'], self.datasets) - self.assertNotIn('data_type_def', spec) - self.assertIs(spec, self.subgroups[0].parent) - self.assertIs(spec, self.subgroups[1].parent) - self.assertIs(spec, self.attributes[0].parent) - self.assertIs(spec, self.attributes[1].parent) - self.assertIs(spec, self.datasets[0].parent) - self.assertIs(spec, self.datasets[1].parent) - json.dumps(spec) - - def test_constructor_nwbtype(self): - spec = GroupSpec('A test group', - name='root_constructor_nwbtype', - datasets=self.datasets, - attributes=self.attributes, - linkable=False, - data_type_def='EphysData') - self.assertFalse(spec['linkable']) - self.assertListEqual(spec['attributes'], self.attributes) - self.assertListEqual(spec['datasets'], self.datasets) - self.assertEqual(spec['data_type_def'], 'EphysData') - self.assertIs(spec, self.attributes[0].parent) - self.assertIs(spec, self.attributes[1].parent) - self.assertIs(spec, self.datasets[0].parent) - self.assertIs(spec, self.datasets[1].parent) - self.assertEqual(spec.data_type_def, 'EphysData') - self.assertIsNone(spec.data_type_inc) - json.dumps(spec) - - def test_set_dataset(self): - spec = GroupSpec('A test group', - name='root_test_set_dataset', - linkable=False, - data_type_def='EphysData') - spec.set_dataset(self.datasets[0]) - self.assertIs(spec, self.datasets[0].parent) - - def test_set_group(self): - spec = GroupSpec('A test group', - name='root_test_set_group', - linkable=False, - data_type_def='EphysData') - spec.set_group(self.subgroups[0]) - spec.set_group(self.subgroups[1]) - self.assertListEqual(spec['groups'], self.subgroups) - self.assertIs(spec, self.subgroups[0].parent) - self.assertIs(spec, self.subgroups[1].parent) - json.dumps(spec) - - def test_type_extension(self): - spec = GroupSpec('A test group', - name='parent_type', - datasets=self.datasets, - attributes=self.attributes, - linkable=False, - data_type_def='EphysData') - dset1_attributes_ext = [ - AttributeSpec('dset1_extra_attribute', 'an extra attribute for the first dataset', 'text') - ] - ext_datasets = [ - DatasetSpec('my first dataset extension', - 'int', - name='dataset1', - attributes=dset1_attributes_ext, - linkable=True), - ] - ext_attributes = [ - AttributeSpec('ext_extra_attribute', 'an extra attribute for the group', 'text'), - ] - ext = GroupSpec('A test group extension', - name='child_type', - datasets=ext_datasets, - attributes=ext_attributes, - linkable=False, - data_type_inc=spec, - data_type_def='SpikeData') - ext_dset1 = ext.get_dataset('dataset1') - ext_dset1_attrs = ext_dset1.attributes - self.assertDictEqual(ext_dset1_attrs[0], dset1_attributes_ext[0]) - self.assertDictEqual(ext_dset1_attrs[1], self.dset1_attributes[0]) - self.assertDictEqual(ext_dset1_attrs[2], self.dset1_attributes[1]) - self.assertEqual(ext.data_type_def, 'SpikeData') - self.assertEqual(ext.data_type_inc, 'EphysData') - - ext_dset2 = ext.get_dataset('dataset2') - self.maxDiff = None - # this will suffice for now, assertDictEqual doesn't do deep equality checks - self.assertEqual(str(ext_dset2), str(self.datasets[1])) - self.assertAttributesEqual(ext_dset2, self.datasets[1]) - - # self.ns_attr_spec - ndt_attr_spec = AttributeSpec('data_type', 'the data type of this object', # noqa: F841 - 'text', value='SpikeData') - - res_attrs = ext.attributes - self.assertDictEqual(res_attrs[0], ext_attributes[0]) - self.assertDictEqual(res_attrs[1], self.attributes[0]) - self.assertDictEqual(res_attrs[2], self.attributes[1]) - - # test that inherited specs are tracked appropriate - for d in self.datasets: - with self.subTest(dataset=d.name): - self.assertTrue(ext.is_inherited_spec(d)) - self.assertFalse(spec.is_inherited_spec(d)) - - json.dumps(spec) - - def assertDatasetsEqual(self, spec1, spec2): - spec1_dsets = spec1.datasets - spec2_dsets = spec2.datasets - if len(spec1_dsets) != len(spec2_dsets): - raise AssertionError('different number of AttributeSpecs') - else: - for i in range(len(spec1_dsets)): - self.assertAttributesEqual(spec1_dsets[i], spec2_dsets[i]) - - def assertAttributesEqual(self, spec1, spec2): - spec1_attr = spec1.attributes - spec2_attr = spec2.attributes - if len(spec1_attr) != len(spec2_attr): - raise AssertionError('different number of AttributeSpecs') - else: - for i in range(len(spec1_attr)): - self.assertDictEqual(spec1_attr[i], spec2_attr[i]) - - def test_add_attribute(self): - spec = GroupSpec('A test group', - name='root_constructor', - groups=self.subgroups, - datasets=self.datasets, - linkable=False) - for attrspec in self.attributes: - spec.add_attribute(**attrspec) - self.assertListEqual(spec['attributes'], self.attributes) - self.assertListEqual(spec['datasets'], self.datasets) - self.assertNotIn('data_type_def', spec) - self.assertIs(spec, self.subgroups[0].parent) - self.assertIs(spec, self.subgroups[1].parent) - self.assertIs(spec, spec.attributes[0].parent) - self.assertIs(spec, spec.attributes[1].parent) - self.assertIs(spec, self.datasets[0].parent) - self.assertIs(spec, self.datasets[1].parent) - json.dumps(spec) - - def test_update_attribute_spec(self): - spec = GroupSpec('A test group', - name='root_constructor', - attributes=[AttributeSpec('attribute1', 'my first attribute', 'text'), ]) - spec.set_attribute(AttributeSpec('attribute1', 'my first attribute', 'int', value=5)) - res = spec.get_attribute('attribute1') - self.assertEqual(res.value, 5) - self.assertEqual(res.dtype, 'int') diff --git a/tests/unit/form_tests/spec_tests/test_load_namespace.py b/tests/unit/form_tests/spec_tests/test_load_namespace.py deleted file mode 100644 index 9e6b1cf9a..000000000 --- a/tests/unit/form_tests/spec_tests/test_load_namespace.py +++ /dev/null @@ -1,114 +0,0 @@ -import unittest2 as unittest -import ruamel.yaml as yaml -import json -import os - -from pynwb.form.spec import AttributeSpec, DatasetSpec, GroupSpec, SpecNamespace, NamespaceCatalog - - -class TestSpecLoad(unittest.TestCase): - NS_NAME = 'test_ns' - - def setUp(self): - self.attributes = [ - AttributeSpec('attribute1', 'my first attribute', 'text'), - AttributeSpec('attribute2', 'my second attribute', 'text') - ] - self.dset1_attributes = [ - AttributeSpec('attribute3', 'my third attribute', 'text'), - AttributeSpec('attribute4', 'my fourth attribute', 'text') - ] - self.dset2_attributes = [ - AttributeSpec('attribute5', 'my fifth attribute', 'text'), - AttributeSpec('attribute6', 'my sixth attribute', 'text') - ] - self.datasets = [ - DatasetSpec('my first dataset', # noqa: F405 - 'int', - name='dataset1', - attributes=self.dset1_attributes, - linkable=True), - DatasetSpec('my second dataset', # noqa: F405 - 'int', - name='dataset2', - dims=(None, None), - attributes=self.dset2_attributes, - linkable=True, - data_type_def='VoltageArray') - ] - self.spec = GroupSpec('A test group', # noqa: F405 - name='root_constructor_nwbtype', - datasets=self.datasets, - attributes=self.attributes, - linkable=False, - data_type_def='EphysData') - dset1_attributes_ext = [ - AttributeSpec('dset1_extra_attribute', 'an extra attribute for the first dataset', 'text') - ] - self.ext_datasets = [ - DatasetSpec('my first dataset extension', # noqa: F405 - 'int', - name='dataset1', - attributes=dset1_attributes_ext, - linkable=True), - ] - self.ext_attributes = [ - AttributeSpec('ext_extra_attribute', 'an extra attribute for the group', 'text'), - ] - self.ext_spec = GroupSpec('A test group extension', # noqa: F405 - name='root_constructor_nwbtype', - datasets=self.ext_datasets, - attributes=self.ext_attributes, - linkable=False, - data_type_inc='EphysData', - data_type_def='SpikeData') - to_dump = {'groups': [self.spec, self.ext_spec]} - self.specs_path = 'test_load_namespace.specs.yaml' - self.namespace_path = 'test_load_namespace.namespace.yaml' - with open(self.specs_path, 'w') as tmp: - yaml.safe_dump(json.loads(json.dumps(to_dump)), tmp, default_flow_style=False) - ns_dict = { - 'doc': 'a test namespace', - 'name': self.NS_NAME, - 'schema': [ - {'source': self.specs_path} - ] - } - self.namespace = SpecNamespace.build_namespace(**ns_dict) # noqa: F405 - to_dump = {'namespaces': [self.namespace]} - with open(self.namespace_path, 'w') as tmp: - yaml.safe_dump(json.loads(json.dumps(to_dump)), tmp, default_flow_style=False) - self.ns_catalog = NamespaceCatalog() # noqa: F405 - - def tearDown(self): - if os.path.exists(self.namespace_path): - os.remove(self.namespace_path) - if os.path.exists(self.specs_path): - os.remove(self.specs_path) - - def test_inherited_attributes(self): - self.ns_catalog.load_namespaces(self.namespace_path, resolve=True) - ts_spec = self.ns_catalog.get_spec(self.NS_NAME, 'EphysData') - es_spec = self.ns_catalog.get_spec(self.NS_NAME, 'SpikeData') - ts_attrs = {s.name for s in ts_spec.attributes} - es_attrs = {s.name for s in es_spec.attributes} - for attr in ts_attrs: - with self.subTest(attr=attr): - self.assertIn(attr, es_attrs) - # self.assertSetEqual(ts_attrs, es_attrs) - ts_dsets = {s.name for s in ts_spec.datasets} - es_dsets = {s.name for s in es_spec.datasets} - for dset in ts_dsets: - with self.subTest(dset=dset): - self.assertIn(dset, es_dsets) - # self.assertSetEqual(ts_dsets, es_dsets) - - def test_inherited_attributes_not_resolved(self): - self.ns_catalog.load_namespaces(self.namespace_path, resolve=False) - es_spec = self.ns_catalog.get_spec(self.NS_NAME, 'SpikeData') - src_attrs = {s.name for s in self.ext_attributes} - ext_attrs = {s.name for s in es_spec.attributes} - self.assertSetEqual(src_attrs, ext_attrs) - src_dsets = {s.name for s in self.ext_datasets} - ext_dsets = {s.name for s in es_spec.datasets} - self.assertSetEqual(src_dsets, ext_dsets) diff --git a/tests/unit/form_tests/spec_tests/test_ref_spec.py b/tests/unit/form_tests/spec_tests/test_ref_spec.py deleted file mode 100644 index 498217f2d..000000000 --- a/tests/unit/form_tests/spec_tests/test_ref_spec.py +++ /dev/null @@ -1,23 +0,0 @@ -import unittest2 as unittest -import json - -from pynwb.form.spec import RefSpec - - -class RefSpecTests(unittest.TestCase): - - def test_constructor(self): - spec = RefSpec('TimeSeries', 'object') - self.assertEqual(spec.target_type, 'TimeSeries') - self.assertEqual(spec.reftype, 'object') - json.dumps(spec) # to ensure there are no circular links - - def test_wrong_reference_type(self): - with self.assertRaises(ValueError): - RefSpec('TimeSeries', 'unknownreftype') - - def test_isregion(self): - spec = RefSpec('TimeSeries', 'object') - self.assertFalse(spec.is_region()) - spec = RefSpec('NWBData', 'region') - self.assertTrue(spec.is_region()) diff --git a/tests/unit/form_tests/spec_tests/test_spec_catalog.py b/tests/unit/form_tests/spec_tests/test_spec_catalog.py deleted file mode 100644 index 293eaac0d..000000000 --- a/tests/unit/form_tests/spec_tests/test_spec_catalog.py +++ /dev/null @@ -1,57 +0,0 @@ -import unittest2 as unittest - -from pynwb.form.spec import GroupSpec, DatasetSpec, AttributeSpec, SpecCatalog - - -class SpecCatalogTest(unittest.TestCase): - - def setUp(self): - self.catalog = SpecCatalog() - self.attributes = [ - AttributeSpec('attribute1', 'my first attribute', 'text'), - AttributeSpec('attribute2', 'my second attribute', 'text') - ] - self.spec = DatasetSpec('my first dataset', - 'int', - name='dataset1', - dims=(None, None), - attributes=self.attributes, - linkable=False, - data_type_def='EphysData') - - def test_register_spec(self): - self.catalog.register_spec(self.spec, 'test.yaml') - result = self.catalog.get_spec('EphysData') - self.assertIs(result, self.spec) - - def test_hierarchy(self): - spikes_spec = DatasetSpec('my extending dataset', 'int', - data_type_inc='EphysData', - data_type_def='SpikeData') - - lfp_spec = DatasetSpec('my second extending dataset', 'int', - data_type_inc='EphysData', - data_type_def='LFPData') - - self.catalog.register_spec(self.spec, 'test.yaml') - self.catalog.register_spec(spikes_spec, 'test.yaml') - self.catalog.register_spec(lfp_spec, 'test.yaml') - - spike_hierarchy = self.catalog.get_hierarchy('SpikeData') - lfp_hierarchy = self.catalog.get_hierarchy('LFPData') - ephys_hierarchy = self.catalog.get_hierarchy('EphysData') - self.assertTupleEqual(spike_hierarchy, ('SpikeData', 'EphysData')) - self.assertTupleEqual(lfp_hierarchy, ('LFPData', 'EphysData')) - self.assertTupleEqual(ephys_hierarchy, ('EphysData',)) - - def test_get_spec_source_file(self): - spikes_spec = GroupSpec('test group', - data_type_def='SpikeData') - source_file_path = '/test/myt/test.yaml' - self.catalog.auto_register(spikes_spec, source_file_path) - recorded_source_file_path = self.catalog.get_spec_source_file('SpikeData') - self.assertEqual(recorded_source_file_path, source_file_path) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/unit/form_tests/test_container.py b/tests/unit/form_tests/test_container.py deleted file mode 100644 index 58ee3ce6f..000000000 --- a/tests/unit/form_tests/test_container.py +++ /dev/null @@ -1,52 +0,0 @@ -import unittest2 as unittest - -from pynwb.form.container import Container - - -class MyTestClass(Container): - - def __init__(self, src, name, parent=None): - super(MyTestClass, self).__init__(src, name, parent=parent) - - def basic_add(self, **kwargs): - return kwargs - - def basic_add2(self, **kwargs): - return kwargs - - def basic_add2_kw(self, **kwargs): - return kwargs - - -class MyTestSubclass(MyTestClass): - - def basic_add(self, **kwargs): - return kwargs - - def basic_add2_kw(self, **kwargs): - return kwargs - - -class TestNWBContainer(unittest.TestCase): - - def test_constructor(self): - """Test that constructor properly sets parent - """ - parent_obj = MyTestClass('test source', 'obj1') - child_obj = MyTestSubclass('test source', 'obj2', parent=parent_obj) - self.assertIs(child_obj.parent, parent_obj) - - def test_set_parent_parent(self): - """Test that parent setter properly sets parent - """ - parent_obj = MyTestClass('test source', 'obj1') - child_obj = MyTestSubclass('test source', 'obj2') - child_obj.parent = parent_obj - self.assertIs(child_obj.parent, parent_obj) - - def test_slash_restriction(self): - self.assertRaises(ValueError, Container, 'bad/name') - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/unit/form_tests/test_io_hdf5.py b/tests/unit/form_tests/test_io_hdf5.py deleted file mode 100644 index 1d0743f2f..000000000 --- a/tests/unit/form_tests/test_io_hdf5.py +++ /dev/null @@ -1,274 +0,0 @@ -import unittest2 as unittest -from datetime import datetime -from dateutil.tz import tzlocal -import os -from h5py import File, Dataset, Reference -from six import text_type - -from pynwb.form.backends.hdf5 import HDF5IO -from pynwb.form.build import GroupBuilder, DatasetBuilder, LinkBuilder, BuildManager - -from pynwb import TimeSeries, get_type_map - -from numbers import Number - -import json -import numpy as np - - -class HDF5Encoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, Dataset): - ret = None - for t in (list, str): - try: - ret = t(obj) - break - except: # noqa: F722 - pass - if ret is None: - return obj - else: - return ret - elif isinstance(obj, np.int64): - return int(obj) - elif isinstance(obj, bytes): - return str(obj) - return json.JSONEncoder.default(self, obj) - - -class GroupBuilderTestCase(unittest.TestCase): - ''' - A TestCase class for comparing GroupBuilders. - ''' - - def __is_scalar(self, obj): - if hasattr(obj, 'shape'): - return len(obj.shape) == 0 - else: - if any(isinstance(obj, t) for t in (int, str, float, bytes, text_type)): - return True - return False - - def __convert_h5_scalar(self, obj): - if isinstance(obj, Dataset): - return obj[...] - return obj - - def __compare_attr_dicts(self, a, b): - reasons = list() - b_keys = set(b.keys()) - for k in a: - if k not in b: - reasons.append("'%s' attribute missing from second dataset" % k) - else: - if a[k] != b[k]: - reasons.append("'%s' attribute on datasets not equal" % k) - b_keys.remove(k) - for k in b_keys: - reasons.append("'%s' attribute missing from first dataset" % k) - return reasons - - def __compare_dataset(self, a, b): - reasons = self.__compare_attr_dicts(a.attributes, b.attributes) - if not self.__compare_data(a.data, b.data): - reasons.append("dataset '%s' not equal" % a.name) - return reasons - - def __compare_data(self, a, b): - if isinstance(a, Number) and isinstance(b, Number): - return a == b - elif isinstance(a, Number) != isinstance(b, Number): - return False - else: - a_scalar = self.__is_scalar(a) - b_scalar = self.__is_scalar(b) - if a_scalar and b_scalar: - return self.__convert_h5_scalar(a_scalar) == self.__convert_h5_scalar(b_scalar) - elif a_scalar != b_scalar: - return False - if len(a) == len(b): - for i in range(len(a)): - if not self.__compare_data(a[i], b[i]): - return False - else: - return False - return True - - def __fmt(self, val): - return "%s (%s)" % (val, type(val)) - - def __assert_helper(self, a, b): - reasons = list() - b_keys = set(b.keys()) - for k, a_sub in a.items(): - if k in b: - b_sub = b[k] - b_keys.remove(k) - if isinstance(a_sub, LinkBuilder) and isinstance(a_sub, LinkBuilder): - a_sub = a_sub['builder'] - b_sub = b_sub['builder'] - elif isinstance(a_sub, LinkBuilder) != isinstance(a_sub, LinkBuilder): - reasons.append('%s != %s' % (a_sub, b_sub)) - if isinstance(a_sub, DatasetBuilder) and isinstance(a_sub, DatasetBuilder): - # if not self.__compare_dataset(a_sub, b_sub): - # reasons.append('%s != %s' % (a_sub, b_sub)) - reasons.extend(self.__compare_dataset(a_sub, b_sub)) - elif isinstance(a_sub, GroupBuilder) and isinstance(a_sub, GroupBuilder): - reasons.extend(self.__assert_helper(a_sub, b_sub)) - else: - equal = None - a_array = isinstance(a_sub, np.ndarray) - b_array = isinstance(b_sub, np.ndarray) - if a_array and b_array: - equal = np.array_equal(a_sub, b_sub) - elif a_array or b_array: - # if strings, convert before comparing - if b_array: - if b_sub.dtype.char in ('S', 'U'): - a_sub = [np.string_(s) for s in a_sub] - else: - if a_sub.dtype.char in ('S', 'U'): - b_sub = [np.string_(s) for s in b_sub] - equal = np.array_equal(a_sub, b_sub) - else: - equal = a_sub == b_sub - if not equal: - reasons.append('%s != %s' % (self.__fmt(a_sub), self.__fmt(b_sub))) - else: - reasons.append("'%s' not in both" % k) - for k in b_keys: - reasons.append("'%s' not in both" % k) - return reasons - - def assertBuilderEqual(self, a, b): - ''' Tests that two GroupBuilders are equal ''' - reasons = self.__assert_helper(a, b) - if len(reasons): - raise AssertionError(', '.join(reasons)) - return True - - -class TestHDF5Writer(GroupBuilderTestCase): - - def setUp(self): - type_map = get_type_map() - self.manager = BuildManager(type_map) - self.path = "test_pynwb_io_hdf5.h5" - self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzlocal()) - self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) - - self.ts_builder = GroupBuilder('test_timeseries', - attributes={'ancestry': 'TimeSeries', - 'neurodata_type': 'TimeSeries', - 'int_array_attribute': [0, 1, 2, 3], - 'str_array_attribute': ['a', 'b', 'c', 'd'], - 'help': 'General purpose TimeSeries'}, - datasets={'data': DatasetBuilder('data', list(range(100, 200, 10)), - attributes={'unit': 'SIunit', - 'conversion': 1.0, - 'resolution': 0.1}), - 'timestamps': DatasetBuilder( - 'timestamps', list(range(10)), - attributes={'unit': 'Seconds', 'interval': 1})}) - self.ts = TimeSeries('test_timeseries', list(range(100, 200, 10)), - unit='SIunit', resolution=0.1, timestamps=list(range(10))) - self.manager.prebuilt(self.ts, self.ts_builder) - self.builder = GroupBuilder( - 'root', - source=self.path, - groups={'acquisition': - GroupBuilder('acquisition', - groups={'timeseries': - GroupBuilder('timeseries', - groups={'test_timeseries': self.ts_builder}), - 'images': GroupBuilder('images')}), - 'analysis': GroupBuilder('analysis'), - 'epochs': GroupBuilder('epochs'), - 'general': GroupBuilder('general'), - 'processing': GroupBuilder('processing', - groups={'test_module': - GroupBuilder('test_module', - links={'test_timeseries_link': - LinkBuilder(self.ts_builder, - 'test_timeseries_link')})}), - 'stimulus': GroupBuilder( - 'stimulus', - groups={'presentation': - GroupBuilder('presentation'), - 'templates': GroupBuilder('templates')})}, - datasets={'file_create_date': DatasetBuilder('file_create_date', [self.create_date.isoformat()]), - 'identifier': DatasetBuilder('identifier', 'TEST123'), - 'session_description': DatasetBuilder('session_description', 'a test NWB File'), - 'nwb_version': DatasetBuilder('nwb_version', '1.0.6'), - 'session_start_time': DatasetBuilder('session_start_time', str(self.start_time))}, - attributes={'neurodata_type': 'NWBFile'}) - - def tearDown(self): - if os.path.exists(self.path): - os.remove(self.path) - - def check_fields(self): - f = File(self.path) - self.assertIn('acquisition', f) - self.assertIn('analysis', f) - self.assertIn('epochs', f) - self.assertIn('general', f) - self.assertIn('processing', f) - self.assertIn('file_create_date', f) - self.assertIn('identifier', f) - self.assertIn('session_description', f) - self.assertIn('nwb_version', f) - self.assertIn('session_start_time', f) - acq = f.get('acquisition') - self.assertIn('images', acq) - self.assertIn('timeseries', acq) - ts = acq.get('timeseries') - self.assertIn('test_timeseries', ts) - return f - - def test_write_builder(self): - writer = HDF5IO(self.path, manager=self.manager, mode='a') - writer.write_builder(self.builder) - writer.close() - self.check_fields() - - def test_write_attribute_reference_container(self): - writer = HDF5IO(self.path, manager=self.manager, mode='a') - self.builder.set_attribute('ref_attribute', self.ts) - writer.write_builder(self.builder) - writer.close() - f = self.check_fields() - self.assertIsInstance(f.attrs['ref_attribute'], Reference) - self.assertEqual(f['acquisition/timeseries/test_timeseries'], f[f.attrs['ref_attribute']]) - - def test_write_attribute_reference_builder(self): - writer = HDF5IO(self.path, manager=self.manager, mode='a') - self.builder.set_attribute('ref_attribute', self.ts_builder) - writer.write_builder(self.builder) - writer.close() - f = self.check_fields() - self.assertIsInstance(f.attrs['ref_attribute'], Reference) - self.assertEqual(f['acquisition/timeseries/test_timeseries'], f[f.attrs['ref_attribute']]) - - def test_write_context_manager(self): - with HDF5IO(self.path, manager=self.manager, mode='a') as writer: - writer.write_builder(self.builder) - self.check_fields() - - def test_read_builder(self): - self.maxDiff = None - io = HDF5IO(self.path, manager=self.manager, mode='a') - io.write_builder(self.builder) - builder = io.read_builder() - self.assertBuilderEqual(builder, self.builder) - io.close() - - def test_overwrite_written(self): - self.maxDiff = None - io = HDF5IO(self.path, manager=self.manager, mode='a') - io.write_builder(self.builder) - builder = io.read_builder() - with self.assertRaisesRegex(ValueError, "cannot change written to not written"): - builder.written = False - io.close() diff --git a/tests/unit/form_tests/test_io_hdf5_h5tools.py b/tests/unit/form_tests/test_io_hdf5_h5tools.py deleted file mode 100644 index 3be44763d..000000000 --- a/tests/unit/form_tests/test_io_hdf5_h5tools.py +++ /dev/null @@ -1,536 +0,0 @@ -import os -import unittest2 as unittest - -from pynwb.form.data_utils import DataChunkIterator -from pynwb.form.backends.hdf5.h5tools import HDF5IO -from pynwb.form.backends.hdf5 import H5DataIO -from pynwb.form.build import DatasetBuilder -from pynwb.form.spec.namespace import NamespaceCatalog -from h5py import SoftLink, HardLink, ExternalLink, File -from pynwb.file import NWBFile -from pynwb.base import TimeSeries -from pynwb import NWBHDF5IO -from pynwb.spec import NWBNamespace, NWBGroupSpec, NWBDatasetSpec - -from pynwb.ecephys import ElectricalSeries - - -import tempfile -import warnings -import numpy as np -from datetime import datetime -from dateutil.tz import tzlocal - - -class H5IOTest(unittest.TestCase): - """Tests for h5tools IO tools""" - - def setUp(self): - self.test_temp_file = tempfile.NamedTemporaryFile() - - # On Windows h5py cannot truncate an open file in write mode. - # The temp file will be closed before h5py truncates it - # and will be removed during the tearDown step. - self.test_temp_file.close() - self.io = HDF5IO(self.test_temp_file.name, mode='a') - self.f = self.io._file - - def tearDown(self): - path = self.f.filename - self.f.close() - os.remove(path) - del self.f - del self.test_temp_file - self.f = None - self.test_temp_file = None - - ########################################## - # __chunked_iter_fill__(...) tests - ########################################## - def test__chunked_iter_fill_iterator_matched_buffer_size(self): - dci = DataChunkIterator(data=range(10), buffer_size=2) - my_dset = HDF5IO.__chunked_iter_fill__(self.f, 'test_dataset', dci) - self.assertListEqual(my_dset[:].tolist(), list(range(10))) - - def test__chunked_iter_fill_iterator_unmatched_buffer_size(self): - dci = DataChunkIterator(data=range(10), buffer_size=3) - my_dset = HDF5IO.__chunked_iter_fill__(self.f, 'test_dataset', dci) - self.assertListEqual(my_dset[:].tolist(), list(range(10))) - - def test__chunked_iter_fill_numpy_matched_buffer_size(self): - a = np.arange(30).reshape(5, 2, 3) - dci = DataChunkIterator(data=a, buffer_size=1) - my_dset = HDF5IO.__chunked_iter_fill__(self.f, 'test_dataset', dci) - self.assertTrue(np.all(my_dset[:] == a)) - self.assertTupleEqual(my_dset.shape, a.shape) - - def test__chunked_iter_fill_numpy_unmatched_buffer_size(self): - a = np.arange(30).reshape(5, 2, 3) - dci = DataChunkIterator(data=a, buffer_size=3) - my_dset = HDF5IO.__chunked_iter_fill__(self.f, 'test_dataset', dci) - self.assertTrue(np.all(my_dset[:] == a)) - self.assertTupleEqual(my_dset.shape, a.shape) - - def test__chunked_iter_fill_list_matched_buffer_size(self): - a = np.arange(30).reshape(5, 2, 3) - dci = DataChunkIterator(data=a.tolist(), buffer_size=1) - my_dset = HDF5IO.__chunked_iter_fill__(self.f, 'test_dataset', dci) - self.assertTrue(np.all(my_dset[:] == a)) - self.assertTupleEqual(my_dset.shape, a.shape) - - def test__chunked_iter_fill_numpy_unmatched_buffer_size(self): # noqa: F811 - a = np.arange(30).reshape(5, 2, 3) - dci = DataChunkIterator(data=a.tolist(), buffer_size=3) - my_dset = HDF5IO.__chunked_iter_fill__(self.f, 'test_dataset', dci) - self.assertTrue(np.all(my_dset[:] == a)) - self.assertTupleEqual(my_dset.shape, a.shape) - - ########################################## - # write_dataset tests: scalars - ########################################## - def test_write_dataset_scalar(self): - a = 10 - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a, attributes={})) - dset = self.f['test_dataset'] - self.assertTupleEqual(dset.shape, ()) - self.assertEqual(dset[()], a) - - def test_write_dataset_string(self): - a = 'test string' - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a, attributes={})) - dset = self.f['test_dataset'] - self.assertTupleEqual(dset.shape, ()) - # self.assertEqual(dset[()].decode('utf-8'), a) - self.assertEqual(dset[()], a) - - ########################################## - # write_dataset tests: lists - ########################################## - def test_write_dataset_list(self): - a = np.arange(30).reshape(5, 2, 3) - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a.tolist(), attributes={})) - dset = self.f['test_dataset'] - self.assertTrue(np.all(dset[:] == a)) - - def test_write_dataset_list_compress(self): - a = H5DataIO(np.arange(30).reshape(5, 2, 3), - compression='gzip', - compression_opts=5, - shuffle=True, - fletcher32=True) - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a, attributes={})) - dset = self.f['test_dataset'] - self.assertTrue(np.all(dset[:] == a.data)) - self.assertEqual(dset.compression, 'gzip') - self.assertEqual(dset.compression_opts, 5) - self.assertEqual(dset.shuffle, True) - self.assertEqual(dset.fletcher32, True) - - def test_write_dataset_list_enable_default_compress(self): - a = H5DataIO(np.arange(30).reshape(5, 2, 3), - compression=True) - self.assertEqual(a.io_settings['compression'], 'gzip') - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a, attributes={})) - dset = self.f['test_dataset'] - self.assertTrue(np.all(dset[:] == a.data)) - self.assertEqual(dset.compression, 'gzip') - - def test_write_dataset_list_disable_default_compress(self): - with warnings.catch_warnings(record=True) as w: - a = H5DataIO(np.arange(30).reshape(5, 2, 3), - compression=False, - compression_opts=5) - self.assertEqual(len(w), 1) # We expect a warning that compression options are being ignored - self.assertFalse('compression_ops' in a.io_settings) - self.assertFalse('compression' in a.io_settings) - - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a, attributes={})) - dset = self.f['test_dataset'] - self.assertTrue(np.all(dset[:] == a.data)) - self.assertEqual(dset.compression, None) - - def test_write_dataset_list_chunked(self): - a = H5DataIO(np.arange(30).reshape(5, 2, 3), - chunks=(1, 1, 3)) - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a, attributes={})) - dset = self.f['test_dataset'] - self.assertTrue(np.all(dset[:] == a.data)) - self.assertEqual(dset.chunks, (1, 1, 3)) - - def test_write_dataset_list_fillvalue(self): - a = H5DataIO(np.arange(20).reshape(5, 4), fillvalue=-1) - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a, attributes={})) - dset = self.f['test_dataset'] - self.assertTrue(np.all(dset[:] == a.data)) - self.assertEqual(dset.fillvalue, -1) - - ########################################## - # write_dataset tests: tables - ########################################## - def test_write_table(self): - cmpd_dt = np.dtype([('a', np.int32), ('b', np.float64)]) - data = np.zeros(10, dtype=cmpd_dt) - data['a'][1] = 101 - data['b'][1] = 0.1 - dt = [{'name': 'a', 'dtype': 'int32', 'doc': 'a column'}, - {'name': 'b', 'dtype': 'float64', 'doc': 'b column'}] - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', data, attributes={}, dtype=dt)) - dset = self.f['test_dataset'] - self.assertEqual(dset['a'].tolist(), data['a'].tolist()) - self.assertEqual(dset['b'].tolist(), data['b'].tolist()) - - def test_write_table_nested(self): - b_cmpd_dt = np.dtype([('c', np.int32), ('d', np.float64)]) - cmpd_dt = np.dtype([('a', np.int32), ('b', b_cmpd_dt)]) - data = np.zeros(10, dtype=cmpd_dt) - data['a'][1] = 101 - data['b']['c'] = 202 - data['b']['d'] = 10.1 - b_dt = [{'name': 'c', 'dtype': 'int32', 'doc': 'c column'}, - {'name': 'd', 'dtype': 'float64', 'doc': 'd column'}] - dt = [{'name': 'a', 'dtype': 'int32', 'doc': 'a column'}, - {'name': 'b', 'dtype': b_dt, 'doc': 'b column'}] - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', data, attributes={}, dtype=dt)) - dset = self.f['test_dataset'] - self.assertEqual(dset['a'].tolist(), data['a'].tolist()) - self.assertEqual(dset['b'].tolist(), data['b'].tolist()) - - ########################################## - # write_dataset tests: Iterable - ########################################## - def test_write_dataset_iterable(self): - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', range(10), attributes={})) - dset = self.f['test_dataset'] - self.assertListEqual(dset[:].tolist(), list(range(10))) - - def test_write_dataset_iterable_multidimensional_array(self): - a = np.arange(30).reshape(5, 2, 3) - aiter = iter(a) - daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2) - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', daiter, attributes={})) - dset = self.f['test_dataset'] - self.assertListEqual(dset[:].tolist(), a.tolist()) - - def test_write_dataset_iterable_multidimensional_array_compression(self): - a = np.arange(30).reshape(5, 2, 3) - aiter = iter(a) - daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2) - wrapped_daiter = H5DataIO(data=daiter, - compression='gzip', - compression_opts=5, - shuffle=True, - fletcher32=True) - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', wrapped_daiter, attributes={})) - dset = self.f['test_dataset'] - self.assertEqual(dset.shape, a.shape) - self.assertListEqual(dset[:].tolist(), a.tolist()) - self.assertEqual(dset.compression, 'gzip') - self.assertEqual(dset.compression_opts, 5) - self.assertEqual(dset.shuffle, True) - self.assertEqual(dset.fletcher32, True) - - ############################################# - # write_dataset tests: data chunk iterator - ############################################# - def test_write_dataset_data_chunk_iterator(self): - dci = DataChunkIterator(data=np.arange(10), buffer_size=2) - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', dci, attributes={})) - dset = self.f['test_dataset'] - self.assertListEqual(dset[:].tolist(), list(range(10))) - - def test_write_dataset_data_chunk_iterator_with_compression(self): - dci = DataChunkIterator(data=np.arange(10), buffer_size=2) - wrapped_dci = H5DataIO(data=dci, - compression='gzip', - compression_opts=5, - shuffle=True, - fletcher32=True, - chunks=(2,)) - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', wrapped_dci, attributes={})) - dset = self.f['test_dataset'] - self.assertListEqual(dset[:].tolist(), list(range(10))) - self.assertEqual(dset.compression, 'gzip') - self.assertEqual(dset.compression_opts, 5) - self.assertEqual(dset.shuffle, True) - self.assertEqual(dset.fletcher32, True) - self.assertEqual(dset.chunks, (2,)) - - def test_pass_through_of_recommended_chunks(self): - - class DC(DataChunkIterator): - def recommended_chunk_shape(self): - return (5, 1, 1) - dci = DC(data=np.arange(30).reshape(5, 2, 3)) - wrapped_dci = H5DataIO(data=dci, - compression='gzip', - compression_opts=5, - shuffle=True, - fletcher32=True) - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', wrapped_dci, attributes={})) - dset = self.f['test_dataset'] - self.assertEqual(dset.chunks, (5, 1, 1)) - self.assertEqual(dset.compression, 'gzip') - self.assertEqual(dset.compression_opts, 5) - self.assertEqual(dset.shuffle, True) - self.assertEqual(dset.fletcher32, True) - - ############################################# - # H5DataIO general - ############################################# - def test_warning_on_non_gzip_compression(self): - # Make sure no warning is issued when using gzip - with warnings.catch_warnings(record=True) as w: - dset = H5DataIO(np.arange(30), - compression='gzip') - self.assertEqual(len(w), 0) - self.assertEqual(dset.io_settings['compression'], 'gzip') - # Make sure no warning is issued when using szip - with warnings.catch_warnings(record=True) as w: - dset = H5DataIO(np.arange(30), - compression='szip') - self.assertEqual(len(w), 1) - self.assertEqual(dset.io_settings['compression'], 'szip') - # Make sure no warning is issued when using lzf - with warnings.catch_warnings(record=True) as w: - dset = H5DataIO(np.arange(30), - compression='lzf') - self.assertEqual(len(w), 1) - self.assertEqual(dset.io_settings['compression'], 'lzf') - - def test_warning_on_linking_of_regular_array(self): - with warnings.catch_warnings(record=True) as w: - dset = H5DataIO(np.arange(30), - link_data=True) - self.assertEqual(len(w), 1) - self.assertEqual(dset.link_data, False) - - def test_warning_on_setting_io_options_on_h5dataset_input(self): - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', np.arange(10), attributes={})) - with warnings.catch_warnings(record=True) as w: - H5DataIO(self.f['test_dataset'], - compression='gzip', - compression_opts=4, - fletcher32=True, - shuffle=True, - maxshape=(10, 20), - chunks=(10,), - fillvalue=100) - self.assertEqual(len(w), 7) - - ############################################# - # Copy/Link h5py.Dataset object - ############################################# - def test_link_h5py_dataset_input(self): - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', np.arange(10), attributes={})) - self.io.write_dataset(self.f, DatasetBuilder('test_softlink', self.f['test_dataset'], attributes={})) - self.assertTrue(isinstance(self.f.get('test_softlink', getlink=True), SoftLink)) - - def test_copy_h5py_dataset_input(self): - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', np.arange(10), attributes={})) - self.io.write_dataset(self.f, - DatasetBuilder('test_copy', self.f['test_dataset'], attributes={}), - link_data=False) - self.assertTrue(isinstance(self.f.get('test_copy', getlink=True), HardLink)) - self.assertListEqual(self.f['test_dataset'][:].tolist(), - self.f['test_copy'][:].tolist()) - - def test_link_h5py_dataset_h5dataio_input(self): - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', np.arange(10), attributes={})) - self.io.write_dataset(self.f, DatasetBuilder('test_softlink', - H5DataIO(data=self.f['test_dataset'], - link_data=True), - attributes={})) - self.assertTrue(isinstance(self.f.get('test_softlink', getlink=True), SoftLink)) - - def test_copy_h5py_dataset_h5dataio_input(self): - self.io.write_dataset(self.f, DatasetBuilder('test_dataset', np.arange(10), attributes={})) - self.io.write_dataset(self.f, - DatasetBuilder('test_copy', - H5DataIO(data=self.f['test_dataset'], - link_data=False), # Force dataset copy - attributes={}), - link_data=True) # Make sure the default behavior is set to link the data - self.assertTrue(isinstance(self.f.get('test_copy', getlink=True), HardLink)) - self.assertListEqual(self.f['test_dataset'][:].tolist(), - self.f['test_copy'][:].tolist()) - - def test_list_fill_empty(self): - dset = self.io.__list_fill__(self.f, 'empty_dataset', [], options={'dtype': int, 'io_settings': {}}) - self.assertTupleEqual(dset.shape, (0,)) - - def test_list_fill_empty_no_dtype(self): - with self.assertRaisesRegex(Exception, r"cannot add \S+ to [/\S]+ - could not determine type"): - self.io.__list_fill__(self.f, 'empty_dataset', []) - - -class TestCacheSpec(unittest.TestCase): - - def test_cache_spec(self): - self.test_temp_file = tempfile.NamedTemporaryFile() - # On Windows h5py cannot truncate an open file in write mode. - # The temp file will be closed before h5py truncates it - # and will be removed during the tearDown step. - self.test_temp_file.close() - self.io = NWBHDF5IO(self.test_temp_file.name, mode='a') - # Setup all the data we need - start_time = datetime(2017, 4, 3, 11, tzinfo=tzlocal()) - create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) - data = np.arange(1000).reshape((100, 10)) - timestamps = np.arange(100) - # Create the first file - nwbfile1 = NWBFile(session_description='demonstrate external files', - identifier='NWBE1', - session_start_time=start_time, - file_create_date=create_date) - - test_ts1 = TimeSeries(name='test_timeseries', - data=data, - unit='SIunit', - timestamps=timestamps) - nwbfile1.add_acquisition(test_ts1) - # Write the first file - self.io.write(nwbfile1, cache_spec=True) - self.io.close() - ns_catalog = NamespaceCatalog(group_spec_cls=NWBGroupSpec, - dataset_spec_cls=NWBDatasetSpec, - spec_namespace_cls=NWBNamespace) - NWBHDF5IO.load_namespaces(ns_catalog, self.test_temp_file.name) - self.assertEqual(ns_catalog.namespaces, ('core',)) - source_types = self.__get_types(self.io.manager.namespace_catalog) - read_types = self.__get_types(ns_catalog) - self.assertSetEqual(source_types, read_types) - - def __get_types(self, catalog): - types = set() - for ns_name in catalog.namespaces: - ns = catalog.get_namespace(ns_name) - for source in ns['schema']: - types.update(catalog.get_types(source['source'])) - return types - - -class TestLinkResolution(unittest.TestCase): - - def test_link_resolve(self): - nwbfile = NWBFile("a file with header data", "NB123A", datetime(2018, 6, 1, tzinfo=tzlocal())) - device = nwbfile.create_device('device_name') - electrode_group = nwbfile.create_electrode_group( - name='electrode_group_name', - description='desc', - device=device, - location='unknown') - nwbfile.add_electrode(id=0, - x=1.0, y=2.0, z=3.0, # position? - imp=2.718, - location='unknown', - filtering='unknown', - group=electrode_group) - etr = nwbfile.create_electrode_table_region([0], 'etr_name') - for passband in ('theta', 'gamma'): - electrical_series = ElectricalSeries(name=passband + '_phase', - data=[1., 2., 3.], - rate=0.0, - electrodes=etr) - nwbfile.add_acquisition(electrical_series) - with NWBHDF5IO(self.path, 'w') as io: - io.write(nwbfile) - with NWBHDF5IO(self.path, 'r') as io: - io.read() - - def setUp(self): - self.path = "test_link_resolve.nwb" - - def tearDown(self): - if os.path.exists(self.path): - os.remove(self.path) - - -class NWBHDF5IOMultiFileTest(unittest.TestCase): - """Tests for h5tools IO tools""" - - def setUp(self): - numfiles = 3 - self.test_temp_files = [tempfile.NamedTemporaryFile() for i in range(numfiles)] - - # On Windows h5py cannot truncate an open file in write mode. - # The temp file will be closed before h5py truncates it - # and will be removed during the tearDown step. - for i in self.test_temp_files: - i.close() - self.io = [NWBHDF5IO(i.name, mode='a') for i in self.test_temp_files] - self.f = [i._file for i in self.io] - - def tearDown(self): - # Close all the files - for i in self.io: - i.close() - del(i) - self.io = None - self.f = None - # Make sure the files have been deleted - for tf in self.test_temp_files: - try: - os.remove(tf.name) - except OSError: - pass - self.test_temp_files = None - - def test_copy_file_with_external_links(self): - # Setup all the data we need - start_time = datetime(2017, 4, 3, 11, tzinfo=tzlocal()) - create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) - data = np.arange(1000).reshape((100, 10)) - timestamps = np.arange(100) - # Create the first file - nwbfile1 = NWBFile(session_description='demonstrate external files', - identifier='NWBE1', - session_start_time=start_time, - file_create_date=create_date) - - test_ts1 = TimeSeries(name='test_timeseries', - data=data, - unit='SIunit', - timestamps=timestamps) - nwbfile1.add_acquisition(test_ts1) - # Write the first file - self.io[0].write(nwbfile1) - nwbfile1_read = self.io[0].read() - - # Create the second file - nwbfile2 = NWBFile(session_description='demonstrate external files', - identifier='NWBE1', - session_start_time=start_time, - file_create_date=create_date) - - test_ts2 = TimeSeries(name='test_timeseries', - data=nwbfile1_read.get_acquisition('test_timeseries').data, - unit='SIunit', - timestamps=timestamps) - nwbfile2.add_acquisition(test_ts2) - # Write the second file - self.io[1].write(nwbfile2) - self.io[1].close() - self.io[0].close() # Don't forget to close the first file too - - # Copy the file - self.io[2].close() - HDF5IO.copy_file(source_filename=self.test_temp_files[1].name, - dest_filename=self.test_temp_files[2].name, - expand_external=True, - expand_soft=False, - expand_refs=False) - - # Test that everything is working as expected - # Confirm that our original data file is correct - f1 = File(self.test_temp_files[0].name) - self.assertTrue(isinstance(f1.get('/acquisition/test_timeseries/data', getlink=True), HardLink)) - # Confirm that we successfully created and External Link in our second file - f2 = File(self.test_temp_files[1].name) - self.assertTrue(isinstance(f2.get('/acquisition/test_timeseries/data', getlink=True), ExternalLink)) - # Confirm that we successfully resolved the External Link when we copied our second file - f3 = File(self.test_temp_files[2].name) - self.assertTrue(isinstance(f3.get('/acquisition/test_timeseries/data', getlink=True), HardLink)) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/unit/form_tests/test_query.py b/tests/unit/form_tests/test_query.py deleted file mode 100644 index ecc7c994c..000000000 --- a/tests/unit/form_tests/test_query.py +++ /dev/null @@ -1,162 +0,0 @@ -import unittest2 as unittest -import os -from h5py import File -import numpy as np - -from pynwb.form.query import FORMDataset, Query -from pynwb.form.array import SortedArray, LinSpace - -from six import with_metaclass -from abc import ABCMeta - - -class AbstractQueryTest(with_metaclass(ABCMeta, unittest.TestCase)): - - def getDataset(self): - raise unittest.SkipTest('getDataset must be implemented') - - def setUp(self): - self.dset = self.getDataset() - self.wrapper = FORMDataset(self.dset) # noqa: F405 - - def test_get_dataset(self): - array = self.wrapper.dataset - self.assertIsInstance(array, SortedArray) # noqa: F405 - - def test___gt__(self): - ''' - Test wrapper greater than magic method - ''' - q = self.wrapper > 5 - self.assertIsInstance(q, Query) # noqa: F405 - result = q.evaluate() - expected = [False, False, False, False, False, - False, True, True, True, True] - expected = slice(6, 10) - self.assertEqual(result, expected) - - def test___ge__(self): - ''' - Test wrapper greater than or equal magic method - ''' - q = self.wrapper >= 5 - self.assertIsInstance(q, Query) # noqa: F405 - result = q.evaluate() - expected = [False, False, False, False, False, - True, True, True, True, True] - expected = slice(5, 10) - self.assertEqual(result, expected) - - def test___lt__(self): - ''' - Test wrapper less than magic method - ''' - q = self.wrapper < 5 - self.assertIsInstance(q, Query) # noqa: F405 - result = q.evaluate() - expected = [True, True, True, True, True, - False, False, False, False, False] - expected = slice(0, 5) - self.assertEqual(result, expected) - - def test___le__(self): - ''' - Test wrapper less than or equal magic method - ''' - q = self.wrapper <= 5 - self.assertIsInstance(q, Query) # noqa: F405 - result = q.evaluate() - expected = [True, True, True, True, True, - True, False, False, False, False] - expected = slice(0, 6) - self.assertEqual(result, expected) - - def test___eq__(self): - ''' - Test wrapper equals magic method - ''' - q = self.wrapper == 5 - self.assertIsInstance(q, Query) # noqa: F405 - result = q.evaluate() - expected = [False, False, False, False, False, - True, False, False, False, False] - expected = 5 - self.assertTrue(np.array_equal(result, expected)) - - def test___ne__(self): - ''' - Test wrapper not equal magic method - ''' - q = self.wrapper != 5 - self.assertIsInstance(q, Query) # noqa: F405 - result = q.evaluate() - expected = [True, True, True, True, True, - False, True, True, True, True] - expected = [slice(0, 5), slice(6, 10)] - self.assertTrue(np.array_equal(result, expected)) - - def test___getitem__(self): - ''' - Test wrapper getitem using slice - ''' - result = self.wrapper[0:5] - expected = [0, 1, 2, 3, 4] - self.assertTrue(np.array_equal(result, expected)) - - def test___getitem__query(self): - ''' - Test wrapper getitem using query - ''' - q = self.wrapper < 5 - result = self.wrapper[q] - expected = [0, 1, 2, 3, 4] - self.assertTrue(np.array_equal(result, expected)) - - -class SortedQueryTest(AbstractQueryTest): - - path = 'SortedQueryTest.h5' - - def getDataset(self): - self.f = File(self.path, 'w') - self.input = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - self.d = self.f.create_dataset('dset', data=self.input) - return SortedArray(self.d) # noqa: F405 - - def tearDown(self): - self.f.close() - if os.path.exists(self.path): - os.remove(self.path) - - -class LinspaceQueryTest(AbstractQueryTest): - - path = 'LinspaceQueryTest.h5' - - def getDataset(self): - return LinSpace(0, 10, 1) # noqa: F405 - - -class CompoundQueryTest(unittest.TestCase): - - def getM(self): - return SortedArray(np.arange(10, 20, 1)) - - def getN(self): - return SortedArray(np.arange(10.0, 20.0, 0.5)) - - def setUp(self): - self.m = FORMDataset(self.getM()) - self.n = FORMDataset(self.getN()) - - @unittest.skip('not implemented') - def test_map(self): - q = self.m == (12, 16) # IN operation - q.evaluate() # [2,3,4,5] - q.evaluate(False) # RangeResult(2,6) - r = self.m[q] # noqa: F841 - r = self.m[q.evaluate()] # noqa: F841 - r = self.m[q.evaluate(False)] # noqa: F841 - - def tearDown(self): - pass diff --git a/tests/unit/form_tests/utils_test/__init__.py b/tests/unit/form_tests/utils_test/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/unit/form_tests/utils_test/test_core.py b/tests/unit/form_tests/utils_test/test_core.py deleted file mode 100644 index 9463ae1aa..000000000 --- a/tests/unit/form_tests/utils_test/test_core.py +++ /dev/null @@ -1,322 +0,0 @@ -import unittest2 as unittest -from six import text_type - -from pynwb.form.utils import docval, fmt_docval_args - - -class MyTestClass(object): - - @docval({'name': 'arg1', 'type': str, 'doc': 'argument1 is a str'}) - def basic_add(self, **kwargs): - return kwargs - - @docval({'name': 'arg1', 'type': str, 'doc': 'argument1 is a str'}, - {'name': 'arg2', 'type': int, 'doc': 'argument2 is a int'}) - def basic_add2(self, **kwargs): - return kwargs - - @docval({'name': 'arg1', 'type': text_type, 'doc': 'argument1 is a str'}, # noqa: F40 - {'name': 'arg2', 'type': int, 'doc': 'argument2 is a int'}) - def basic_add2_text_type(self, **kwargs): - return kwargs - - @docval({'name': 'arg1', 'type': str, 'doc': 'argument1 is a str'}, - {'name': 'arg2', 'type': 'int', 'doc': 'argument2 is a int'}, - {'name': 'arg3', 'type': bool, 'doc': 'argument3 is a bool. it defaults to False', 'default': False}) - def basic_add2_kw(self, **kwargs): - return kwargs - - @docval({'name': 'arg1', 'type': str, 'doc': 'argument1 is a str', 'default': 'a'}, - {'name': 'arg2', 'type': int, 'doc': 'argument2 is a int', 'default': 1}) - def basic_only_kw(self, **kwargs): - return kwargs - - -class MyTestSubclass(MyTestClass): - - @docval({'name': 'arg1', 'type': str, 'doc': 'argument1 is a str'}, - {'name': 'arg2', 'type': int, 'doc': 'argument2 is a int'}) - def basic_add(self, **kwargs): - return kwargs - - @docval({'name': 'arg1', 'type': str, 'doc': 'argument1 is a str'}, - {'name': 'arg2', 'type': int, 'doc': 'argument2 is a int'}, - {'name': 'arg3', 'type': bool, 'doc': 'argument3 is a bool. it defaults to False', 'default': False}, - {'name': 'arg4', 'type': str, 'doc': 'argument4 is a str'}, - {'name': 'arg5', 'type': 'float', 'doc': 'argument5 is a float'}, - {'name': 'arg6', 'type': bool, 'doc': 'argument6 is a bool. it defaults to False', 'default': None}) - def basic_add2_kw(self, **kwargs): - return kwargs - - -class TestDocValidator(unittest.TestCase): - - def setUp(self): - self.test_obj = MyTestClass() - self.test_obj_sub = MyTestSubclass() - - def test_bad_type(self): - exp_msg = "argtype must be a type, a str, a list, a tuple, or None - got " - with self.assertRaisesRegex(ValueError, exp_msg): - @docval({'name': 'arg1', 'type': {'a': 1}, 'doc': 'this is a bad type'}) - def method(self, **kwargs): - pass - method(self, arg1=1234560) - - def test_bad_shape(self): - @docval({'name': 'arg1', 'type': 'array_data', 'doc': 'this is a bad shape', 'shape': (None, 2)}) - def method(self, **kwargs): - pass - with self.assertRaises(ValueError): - method(self, arg1=[[1]]) - with self.assertRaises(ValueError): - method(self, arg1=[1]) - # this should work - method(self, arg1=[[1, 1]]) - - def test_multi_shape(self): - @docval({'name': 'arg1', 'type': 'array_data', 'doc': 'this is a bad shape', - 'shape': ((None,), (None, 2))}) - def method1(self, **kwargs): - pass - - method1(self, arg1=[[1, 1]]) - method1(self, arg1=[1, 2]) - with self.assertRaises(ValueError): - method1(self, arg1=[[1, 1, 1]]) - - def test_fmt_docval_args(self): - """ Test that fmt_docval_args works """ - test_kwargs = { - 'arg1': 'a string', - 'arg2': 1, - 'arg3': True, - } - rec_args, rec_kwargs = fmt_docval_args(self.test_obj.basic_add2_kw, test_kwargs) - exp_args = ['a string', 1] - self.assertListEqual(rec_args, exp_args) - exp_kwargs = {'arg3': True} - self.assertDictEqual(rec_kwargs, exp_kwargs) - - def test_docval_add(self): - """Test that docval works with a single positional - argument - """ - kwargs = self.test_obj.basic_add('a string') - self.assertDictEqual(kwargs, {'arg1': 'a string'}) - - def test_docval_add_kw(self): - """Test that docval works with a single positional - argument passed as key-value - """ - kwargs = self.test_obj.basic_add(arg1='a string') - self.assertDictEqual(kwargs, {'arg1': 'a string'}) - - def test_docval_add_missing_args(self): - """Test that docval catches missing argument - with a single positional argument - """ - with self.assertRaises(TypeError) as cm: - kwargs = self.test_obj.basic_add() # noqa: F841 - msg = "missing argument 'arg1'" - self.assertEqual(cm.exception.args[0], msg) - - def test_docval_add2(self): - """Test that docval works with two positional - arguments - """ - kwargs = self.test_obj.basic_add2('a string', 100) - self.assertDictEqual(kwargs, {'arg1': 'a string', 'arg2': 100}) - - def test_docval_add2_text_type_w_str(self): - """Test that docval works with two positional - arguments - """ - kwargs = self.test_obj.basic_add2_text_type('a string', 100) - self.assertDictEqual(kwargs, {'arg1': 'a string', 'arg2': 100}) - - def test_docval_add2_text_type_w_unicode(self): - """Test that docval works with two positional - arguments - """ - kwargs = self.test_obj.basic_add2_text_type(u'a string', 100) - self.assertDictEqual(kwargs, {'arg1': u'a string', 'arg2': 100}) - - def test_docval_add2_kw_default(self): - """Test that docval works with two positional - arguments and a keyword argument when using - default keyword argument value - """ - kwargs = self.test_obj.basic_add2_kw('a string', 100) - self.assertDictEqual(kwargs, {'arg1': 'a string', 'arg2': 100, 'arg3': False}) - - def test_docval_add2_pos_as_kw(self): - """Test that docval works with two positional - arguments and a keyword argument when using - default keyword argument value, but pass - positional arguments by key-value - """ - kwargs = self.test_obj.basic_add2_kw(arg1='a string', arg2=100) - self.assertDictEqual(kwargs, {'arg1': 'a string', 'arg2': 100, 'arg3': False}) - - def test_docval_add2_kw_kw_syntax(self): - """Test that docval works with two positional - arguments and a keyword argument when specifying - keyword argument value with keyword syntax - """ - kwargs = self.test_obj.basic_add2_kw('a string', 100, arg3=True) - self.assertDictEqual(kwargs, {'arg1': 'a string', 'arg2': 100, 'arg3': True}) - - def test_docval_add2_kw_all_kw_syntax(self): - """Test that docval works with two positional - arguments and a keyword argument when specifying - all arguments by key-value - """ - kwargs = self.test_obj.basic_add2_kw(arg1='a string', arg2=100, arg3=True) - self.assertDictEqual(kwargs, {'arg1': 'a string', 'arg2': 100, 'arg3': True}) - - def test_docval_add2_kw_pos_syntax(self): - """Test that docval works with two positional - arguments and a keyword argument when specifying - keyword argument value with positional syntax - """ - kwargs = self.test_obj.basic_add2_kw('a string', 100, True) - self.assertDictEqual(kwargs, {'arg1': 'a string', 'arg2': 100, 'arg3': True}) - - def test_docval_add2_kw_pos_syntax_missing_args(self): - """Test that docval catches incorrect type with two positional - arguments and a keyword argument when specifying - keyword argument value with positional syntax - """ - with self.assertRaises(TypeError) as cm: - kwargs = self.test_obj.basic_add2_kw('a string', 'bad string') # noqa: F841 - - self.assertEqual(cm.exception.args[0], u"incorrect type for 'arg2' (got 'str', expected 'int')") - - def test_docval_add_sub(self): - """Test that docval works with a two positional arguments, - where the second is specified by the subclass implementation - """ - kwargs = self.test_obj_sub.basic_add('a string', 100) - expected = {'arg1': 'a string', 'arg2': 100} - self.assertDictEqual(kwargs, expected) - - def test_docval_add2_kw_default_sub(self): - """Test that docval works with a four positional arguments and - two keyword arguments, where two positional and one keyword - argument is specified in both the parent and sublcass implementations - """ - kwargs = self.test_obj_sub.basic_add2_kw('a string', 100, 'another string', 200.0) - expected = {'arg1': 'a string', 'arg2': 100, - 'arg4': 'another string', 'arg5': 200.0, - 'arg3': False, 'arg6': None} - self.assertDictEqual(kwargs, expected) - - def test_docval_add2_kw_default_sub_missing_args(self): - """Test that docval catches missing arguments with a four positional arguments - and two keyword arguments, where two positional and one keyword - argument is specified in both the parent and sublcass implementations, - when using default values for keyword arguments - """ - with self.assertRaises(TypeError) as cm: - kwargs = self.test_obj_sub.basic_add2_kw('a string', 100, 'another string') # noqa: F841 - msg = "missing argument 'arg5'" - self.assertEqual(cm.exception.args[0], msg) - - def test_docval_add2_kw_kwsyntax_sub(self): - """Test that docval works when called with a four positional - arguments and two keyword arguments, where two positional - and one keyword argument is specified in both the parent - and sublcass implementations - """ - kwargs = self.test_obj_sub.basic_add2_kw('a string', 100, 'another string', 200.0, arg6=True) - expected = {'arg1': 'a string', 'arg2': 100, - 'arg4': 'another string', 'arg5': 200.0, - 'arg3': False, 'arg6': True} - self.assertDictEqual(kwargs, expected) - - def test_docval_add2_kw_kwsyntax_sub_missing_args(self): - """Test that docval catches missing arguments when called with a four positional - arguments and two keyword arguments, where two positional and one keyword - argument is specified in both the parent and sublcass implementations - """ - with self.assertRaises(TypeError) as cm: - kwargs = self.test_obj_sub.basic_add2_kw('a string', 100, 'another string', arg6=True) # noqa: F841 - msg = "missing argument 'arg5'" - self.assertEqual(cm.exception.args[0], msg) - - def test_docval_add2_kw_kwsyntax_sub_nonetype_arg(self): - """Test that docval catches NoneType when called with a four positional - arguments and two keyword arguments, where two positional and one keyword - argument is specified in both the parent and sublcass implementations - """ - with self.assertRaises(TypeError) as cm: - kwargs = self.test_obj_sub.basic_add2_kw('a string', 100, 'another string', None, arg6=True) # noqa: F841 - msg = "incorrect type for 'arg5' (got 'NoneType', expected 'float')" - self.assertEqual(cm.exception.args[0], msg) - - def test_only_kw_no_args(self): - """Test that docval parses arguments when only keyword - arguments exist, and no arguments are specified - """ - kwargs = self.test_obj.basic_only_kw() - self.assertDictEqual(kwargs, {'arg1': 'a', 'arg2': 1}) - - def test_only_kw_arg1_no_arg2(self): - """Test that docval parses arguments when only keyword - arguments exist, and only first argument is specified - as key-value - """ - kwargs = self.test_obj.basic_only_kw(arg1='b') - self.assertDictEqual(kwargs, {'arg1': 'b', 'arg2': 1}) - - def test_only_kw_arg1_pos_no_arg2(self): - """Test that docval parses arguments when only keyword - arguments exist, and only first argument is specified - as positional argument - """ - kwargs = self.test_obj.basic_only_kw('b') - self.assertDictEqual(kwargs, {'arg1': 'b', 'arg2': 1}) - - def test_only_kw_arg2_no_arg1(self): - """Test that docval parses arguments when only keyword - arguments exist, and only second argument is specified - as key-value - """ - kwargs = self.test_obj.basic_only_kw(arg2=2) - self.assertDictEqual(kwargs, {'arg1': 'a', 'arg2': 2}) - - def test_only_kw_arg1_arg2(self): - """Test that docval parses arguments when only keyword - arguments exist, and both arguments are specified - as key-value - """ - kwargs = self.test_obj.basic_only_kw(arg1='b', arg2=2) - self.assertDictEqual(kwargs, {'arg1': 'b', 'arg2': 2}) - - def test_only_kw_arg1_arg2_pos(self): - """Test that docval parses arguments when only keyword - arguments exist, and both arguments are specified - as positional arguments - """ - kwargs = self.test_obj.basic_only_kw('b', 2) - self.assertDictEqual(kwargs, {'arg1': 'b', 'arg2': 2}) - - def test_extra_kwarg(self): - """Test that docval parses arguments when only keyword - arguments exist, and both arguments are specified - as positional arguments - """ - with self.assertRaises(TypeError): - self.test_obj.basic_add2_kw('a string', 100, bar=1000) - - def test_unsupported_docval_term(self): - @docval({'name': 'arg1', 'type': 'array_data', 'doc': 'this is a bad shape', 'unsupported': 'hi!'}) - def method(self, **kwargs): - pass - with self.assertRaises(ValueError): - method(self, arg1=[[1, 1]]) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/unit/form_tests/utils_test/test_core_DataChunkIterator.py b/tests/unit/form_tests/utils_test/test_core_DataChunkIterator.py deleted file mode 100644 index 7e97b7fd2..000000000 --- a/tests/unit/form_tests/utils_test/test_core_DataChunkIterator.py +++ /dev/null @@ -1,129 +0,0 @@ -import unittest2 as unittest - -from pynwb.form.data_utils import DataChunkIterator, DataChunk -import numpy as np - - -class DataChunkIteratorTests(unittest.TestCase): - - def setUp(self): - pass - - def tearDown(self): - pass - - def test_none_iter(self): - dci = DataChunkIterator(None) - self.assertIsNone(dci.maxshape) - self.assertIsNone(dci.dtype) - count = 0 - for chunk in dci: - self.assertEqual(chunk.data, None) - self.assertEqual(chunk.selection, None) - count += 1 - self.assertEqual(count, 0) - self.assertIsNone(dci.recommended_data_shape()) - self.assertIsNone(dci.recommended_chunk_shape()) - - """ def test_numpy_iter_unbuffered(self): - a = np.arange(20).reshape(10,2) - dci = DataChunkIterator(data=a, buffer_size=1) - count = 0 - for v, l in dci: - self.assertEqual(v.shape[0], 1) - self.assertEqual(v.shape[1], 2) - count+=1 - self.assertEqual(count, 10) - self.assertTupleEqual(dci.recommended_data_shape(), a.shape) - self.assertIsNone(dci.recommended_chunk_shape())""" - - def test_numpy_iter_unmatched_buffer_size(self): - a = np.arange(10) - dci = DataChunkIterator(data=a, buffer_size=3) - self.assertTupleEqual(dci.maxshape, a.shape) - self.assertEqual(dci.dtype, a.dtype) - count = 0 - for chunk in dci: - if count < 3: - self.assertEqual(chunk.data.shape[0], 3) - else: - self.assertEqual(chunk.data.shape[0], 1) - count += 1 - self.assertEqual(count, 4) - self.assertTupleEqual(dci.recommended_data_shape(), a.shape) - self.assertIsNone(dci.recommended_chunk_shape()) - - def test_standard_iterator_unbuffered(self): - dci = DataChunkIterator(data=range(10), buffer_size=1) - self.assertEqual(dci.dtype, np.dtype(int)) - self.assertTupleEqual(dci.maxshape, (10,)) - self.assertTupleEqual(dci.recommended_data_shape(), (10,)) # Test before and after iteration - count = 0 - for chunk in dci: - self.assertEqual(chunk.data.shape[0], 1) - count += 1 - self.assertEqual(count, 10) - self.assertTupleEqual(dci.recommended_data_shape(), (10,)) # Test before and after iteration - self.assertIsNone(dci.recommended_chunk_shape()) - - def test_standard_iterator_unmatched_buffersized(self): - dci = DataChunkIterator(data=range(10), buffer_size=3) - self.assertEqual(dci.dtype, np.dtype(int)) - self.assertTupleEqual(dci.maxshape, (10,)) - self.assertIsNone(dci.recommended_chunk_shape()) - self.assertTupleEqual(dci.recommended_data_shape(), (10,)) # Test before and after iteration - count = 0 - for chunk in dci: - if count < 3: - self.assertEqual(chunk.data.shape[0], 3) - else: - self.assertEqual(chunk.data.shape[0], 1) - count += 1 - self.assertTupleEqual(dci.recommended_data_shape(), (10,)) # Test before and after iteration - self.assertEqual(count, 4) - - def test_multidimensional_list(self): - a = np.arange(30).reshape(5, 2, 3).tolist() - dci = DataChunkIterator(a) - self.assertTupleEqual(dci.maxshape, (5, 2, 3)) - self.assertEqual(dci.dtype, np.dtype(int)) - count = 0 - for chunk in dci: - self.assertTupleEqual(chunk.data.shape, (1, 2, 3)) - count += 1 - self.assertEqual(count, 5) - self.assertTupleEqual(dci.recommended_data_shape(), (5, 2, 3)) - self.assertIsNone(dci.recommended_chunk_shape()) - - def test_maxshape(self): - a = np.arange(30).reshape(5, 2, 3) - aiter = iter(a) - daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2) - self.assertEqual(daiter.maxshape, (None, 2, 3)) - - def test_dtype(self): - a = np.arange(30, dtype='int32').reshape(5, 2, 3) - aiter = iter(a) - daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2) - self.assertEqual(daiter.dtype, a.dtype) - - -class DataChunkTests(unittest.TestCase): - - def setUp(self): - pass - - def tearDown(self): - pass - - def test_len_operator_no_data(self): - temp = DataChunk() - self.assertEqual(len(temp), 0) - - def test_len_operator_with_data(self): - temp = DataChunk(np.arange(10).reshape(5, 2)) - self.assertEqual(len(temp), 5) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/unit/form_tests/utils_test/test_core_ShapeValidator.py b/tests/unit/form_tests/utils_test/test_core_ShapeValidator.py deleted file mode 100644 index 74a69641a..000000000 --- a/tests/unit/form_tests/utils_test/test_core_ShapeValidator.py +++ /dev/null @@ -1,198 +0,0 @@ -import unittest2 as unittest - -from pynwb.form.data_utils import ShapeValidatorResult, DataChunkIterator, assertEqualShape -import numpy as np - - -class ShapeValidatorTests(unittest.TestCase): - - def setUp(self): - pass - - def tearDown(self): - pass - - def test_array_all_dimensions_match(self): - # Test match - d1 = np.arange(10).reshape(2, 5) - d2 = np.arange(10).reshape(2, 5) - res = assertEqualShape(d1, d2) - self.assertTrue(res.result) - self.assertIsNone(res.error) - self.assertTupleEqual(res.ignored, ()) - self.assertTupleEqual(res.unmatched, ()) - self.assertTupleEqual(res.shape1, (2, 5)) - self.assertTupleEqual(res.shape2, (2, 5)) - self.assertTupleEqual(res.axes1, (0, 1)) - self.assertTupleEqual(res.axes2, (0, 1)) - - def test_array_dimensions_mismatch(self): - # Test unmatched - d1 = np.arange(10).reshape(2, 5) - d2 = np.arange(10).reshape(5, 2) - res = assertEqualShape(d1, d2) - self.assertFalse(res.result) - self.assertEqual(res.error, 'AXIS_LEN_ERROR') - self.assertTupleEqual(res.ignored, ()) - self.assertTupleEqual(res.unmatched, ((0, 0), (1, 1))) - self.assertTupleEqual(res.shape1, (2, 5)) - self.assertTupleEqual(res.shape2, (5, 2)) - self.assertTupleEqual(res.axes1, (0, 1)) - self.assertTupleEqual(res.axes2, (0, 1)) - - def test_array_unequal_number_of_dimensions(self): - # Test unequal num dims - d1 = np.arange(10).reshape(2, 5) - d2 = np.arange(20).reshape(5, 2, 2) - res = assertEqualShape(d1, d2) - self.assertFalse(res.result) - self.assertEqual(res.error, 'NUM_AXES_ERROR') - self.assertTupleEqual(res.ignored, ()) - self.assertTupleEqual(res.unmatched, ()) - self.assertTupleEqual(res.shape1, (2, 5)) - self.assertTupleEqual(res.shape2, (5, 2, 2)) - self.assertTupleEqual(res.axes1, (0, 1)) - self.assertTupleEqual(res.axes2, (0, 1, 2)) - - def test_array_unequal_number_of_dimensions_check_one_axis_only(self): - # Test unequal num dims compare one axis - d1 = np.arange(10).reshape(2, 5) - d2 = np.arange(20).reshape(2, 5, 2) - res = assertEqualShape(d1, d2, 0, 0) - self.assertTrue(res.result) - self.assertIsNone(res.error) - self.assertTupleEqual(res.ignored, ()) - self.assertTupleEqual(res.unmatched, ()) - self.assertTupleEqual(res.shape1, (2, 5)) - self.assertTupleEqual(res.shape2, (2, 5, 2)) - self.assertTupleEqual(res.axes1, (0,)) - self.assertTupleEqual(res.axes2, (0,)) - - def test_array_unequal_number_of_dimensions_check_multiple_axesy(self): - # Test unequal num dims compare multiple axes - d1 = np.arange(10).reshape(2, 5) - d2 = np.arange(20).reshape(5, 2, 2) - res = assertEqualShape(d1, d2, [0, 1], [1, 0]) - self.assertTrue(res.result) - self.assertIsNone(res.error) - self.assertTupleEqual(res.ignored, ()) - self.assertTupleEqual(res.unmatched, ()) - self.assertTupleEqual(res.shape1, (2, 5)) - self.assertTupleEqual(res.shape2, (5, 2, 2)) - self.assertTupleEqual(res.axes1, (0, 1)) - self.assertTupleEqual(res.axes2, (1, 0)) - - def test_array_unequal_number_of_axes_for_comparison(self): - # Test unequal num axes for comparison - d1 = np.arange(10).reshape(2, 5) - d2 = np.arange(20).reshape(5, 2, 2) - res = assertEqualShape(d1, d2, [0, 1], 1) - self.assertFalse(res.result) - self.assertEqual(res.error, "NUM_AXES_ERROR") - self.assertTupleEqual(res.ignored, ()) - self.assertTupleEqual(res.unmatched, ()) - self.assertTupleEqual(res.shape1, (2, 5)) - self.assertTupleEqual(res.shape2, (5, 2, 2)) - self.assertTupleEqual(res.axes1, (0, 1)) - self.assertTupleEqual(res.axes2, (1,)) - - def test_array_axis_index_out_of_bounds_single_axis(self): - # Test too large frist axis - d1 = np.arange(10).reshape(2, 5) - d2 = np.arange(20).reshape(5, 2, 2) - res = assertEqualShape(d1, d2, 4, 1) - self.assertFalse(res.result) - self.assertEqual(res.error, 'AXIS_OUT_OF_BOUNDS') - self.assertTupleEqual(res.ignored, ()) - self.assertTupleEqual(res.unmatched, ()) - self.assertTupleEqual(res.shape1, (2, 5)) - self.assertTupleEqual(res.shape2, (5, 2, 2)) - self.assertTupleEqual(res.axes1, (4,)) - self.assertTupleEqual(res.axes2, (1,)) - - def test_array_axis_index_out_of_bounds_mutilple_axis(self): - # Test too large second axis - d1 = np.arange(10).reshape(2, 5) - d2 = np.arange(20).reshape(5, 2, 2) - res = assertEqualShape(d1, d2, [0, 1], [5, 0]) - self.assertFalse(res.result) - self.assertEqual(res.error, 'AXIS_OUT_OF_BOUNDS') - self.assertTupleEqual(res.ignored, ()) - self.assertTupleEqual(res.unmatched, ()) - self.assertTupleEqual(res.shape1, (2, 5)) - self.assertTupleEqual(res.shape2, (5, 2, 2)) - self.assertTupleEqual(res.axes1, (0, 1)) - self.assertTupleEqual(res.axes2, (5, 0)) - - def test_DataChunkIterators_match(self): - # Compare data chunk iterators - d1 = DataChunkIterator(data=np.arange(10).reshape(2, 5)) - d2 = DataChunkIterator(data=np.arange(10).reshape(2, 5)) - res = assertEqualShape(d1, d2) - self.assertTrue(res.result) - self.assertIsNone(res.error) - self.assertTupleEqual(res.ignored, ()) - self.assertTupleEqual(res.unmatched, ()) - self.assertTupleEqual(res.shape1, (2, 5)) - self.assertTupleEqual(res.shape2, (2, 5)) - self.assertTupleEqual(res.axes1, (0, 1)) - self.assertTupleEqual(res.axes2, (0, 1)) - - def test_DataChunkIterator_ignore_undetermined_axis(self): - # Compare data chunk iterators with undetermined axis (ignore axis) - d1 = DataChunkIterator(data=np.arange(10).reshape(2, 5), maxshape=(None, 5)) - d2 = DataChunkIterator(data=np.arange(10).reshape(2, 5)) - res = assertEqualShape(d1, d2, ignore_undetermined=True) - self.assertTrue(res.result) - self.assertIsNone(res.error) - self.assertTupleEqual(res.ignored, ((0, 0),)) - self.assertTupleEqual(res.unmatched, ()) - self.assertTupleEqual(res.shape1, (None, 5)) - self.assertTupleEqual(res.shape2, (2, 5)) - self.assertTupleEqual(res.axes1, (0, 1)) - self.assertTupleEqual(res.axes2, (0, 1)) - - def test_DataChunkIterator_error_on_undetermined_axis(self): - # Compare data chunk iterators with undetermined axis (error on undetermined axis) - d1 = DataChunkIterator(data=np.arange(10).reshape(2, 5), maxshape=(None, 5)) - d2 = DataChunkIterator(data=np.arange(10).reshape(2, 5)) - res = assertEqualShape(d1, d2, ignore_undetermined=False) - self.assertFalse(res.result) - self.assertEqual(res.error, 'AXIS_LEN_ERROR') - self.assertTupleEqual(res.ignored, ()) - self.assertTupleEqual(res.unmatched, ((0, 0),)) - self.assertTupleEqual(res.shape1, (None, 5)) - self.assertTupleEqual(res.shape2, (2, 5)) - self.assertTupleEqual(res.axes1, (0, 1)) - self.assertTupleEqual(res.axes2, (0, 1)) - - -class ShapeValidatorResultTests(unittest.TestCase): - - def setUp(self): - pass - - def tearDown(self): - pass - - def test_default_message(self): - temp = ShapeValidatorResult() - temp.error = 'AXIS_LEN_ERROR' - self.assertEqual(temp.default_message, ShapeValidatorResult.SHAPE_ERROR[temp.error]) - - def test_set_error_to_illegal_type(self): - temp = ShapeValidatorResult() - with self.assertRaises(ValueError): - temp.error = 'MY_ILLEGAL_ERROR_TYPE' - - def test_ensure_use_of_tuples_during_asignment(self): - temp = ShapeValidatorResult() - temp_d = [1, 2] - temp_cases = ['shape1', 'shape2', 'axes1', 'axes2', 'ignored', 'unmatched'] - for var in temp_cases: - setattr(temp, var, temp_d) - self.assertIsInstance(getattr(temp, var), tuple, var) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/unit/form_tests/validator_tests/__init__.py b/tests/unit/form_tests/validator_tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/unit/form_tests/validator_tests/test_validate.py b/tests/unit/form_tests/validator_tests/test_validate.py deleted file mode 100644 index c0358e563..000000000 --- a/tests/unit/form_tests/validator_tests/test_validate.py +++ /dev/null @@ -1,156 +0,0 @@ -import unittest2 as unittest -from abc import ABCMeta, abstractmethod -from six import with_metaclass -from six import text_type as text - -from pynwb.form.spec import GroupSpec, AttributeSpec, DatasetSpec, SpecCatalog, SpecNamespace -from pynwb.form.build import GroupBuilder, DatasetBuilder -from pynwb.form.validate import ValidatorMap -from pynwb.form.validate.errors import * # noqa: F403 - -CORE_NAMESPACE = 'test_core' - - -class ValidatorTestBase(with_metaclass(ABCMeta, unittest.TestCase)): - - def setUp(self): - spec_catalog = SpecCatalog() - for spec in self.getSpecs(): - spec_catalog.register_spec(spec, 'test.yaml') - self.namespace = SpecNamespace( - 'a test namespace', CORE_NAMESPACE, [{'source': 'test.yaml'}], catalog=spec_catalog) - self.vmap = ValidatorMap(self.namespace) - - @abstractmethod - def getSpecs(self): - pass - - -class TestEmptySpec(ValidatorTestBase): - - def getSpecs(self): - return (GroupSpec('A test group specification with a data type', data_type_def='Bar'),) - - def test_valid(self): - builder = GroupBuilder('my_bar', attributes={'data_type': 'Bar'}) - validator = self.vmap.get_validator('Bar') - result = validator.validate(builder) - self.assertEqual(len(result), 0) - - def test_invalid_missing_req_type(self): - builder = GroupBuilder('my_bar') - err_msg = "builder must have data type defined with attribute '[A-Za-z_]+'" - with self.assertRaisesRegex(ValueError, err_msg): - result = self.vmap.validate(builder) # noqa: F841 - - -class TestBasicSpec(ValidatorTestBase): - - def getSpecs(self): - ret = GroupSpec('A test group specification with a data type', - data_type_def='Bar', - datasets=[DatasetSpec('an example dataset', 'int', name='data', - attributes=[AttributeSpec( - 'attr2', 'an example integer attribute', 'int')])], - attributes=[AttributeSpec('attr1', 'an example string attribute', 'text')]) - return (ret,) - - def test_invalid_missing(self): - builder = GroupBuilder('my_bar', attributes={'data_type': 'Bar'}) - validator = self.vmap.get_validator('Bar') - result = validator.validate(builder) - self.assertEqual(len(result), 2) - self.assertIsInstance(result[0], MissingError) # noqa: F405 - self.assertEqual(result[0].name, 'Bar/attr1') - self.assertIsInstance(result[1], MissingError) # noqa: F405 - self.assertEqual(result[1].name, 'Bar/data') - - def test_invalid_incorrect_type_get_validator(self): - builder = GroupBuilder('my_bar', attributes={'data_type': 'Bar', 'attr1': 10}) - validator = self.vmap.get_validator('Bar') - result = validator.validate(builder) - self.assertEqual(len(result), 2) - self.assertIsInstance(result[0], DtypeError) # noqa: F405 - self.assertEqual(result[0].name, 'Bar/attr1') - self.assertIsInstance(result[1], MissingError) # noqa: F405 - self.assertEqual(result[1].name, 'Bar/data') - - def test_invalid_incorrect_type_validate(self): - builder = GroupBuilder('my_bar', attributes={'data_type': 'Bar', 'attr1': 10}) - result = self.vmap.validate(builder) - self.assertEqual(len(result), 2) - self.assertIsInstance(result[0], DtypeError) # noqa: F405 - self.assertEqual(result[0].name, 'Bar/attr1') - self.assertIsInstance(result[1], MissingError) # noqa: F405 - self.assertEqual(result[1].name, 'Bar/data') - - def test_valid(self): - builder = GroupBuilder('my_bar', - attributes={'data_type': 'Bar', 'attr1': text('a string attribute')}, - datasets=[DatasetBuilder('data', 100, attributes={'attr2': 10})]) - validator = self.vmap.get_validator('Bar') - result = validator.validate(builder) - self.assertEqual(len(result), 0) - - -class TestNestedTypes(ValidatorTestBase): - - def getSpecs(self): - bar = GroupSpec('A test group specification with a data type', - data_type_def='Bar', - datasets=[DatasetSpec('an example dataset', 'int', name='data', - attributes=[AttributeSpec('attr2', 'an example integer attribute', - 'int')])], - attributes=[AttributeSpec('attr1', text('an example string attribute'), 'text')]) - foo = GroupSpec('A test group that contains a data type', - data_type_def='Foo', - groups=[GroupSpec('A Bar group for Foos', name='my_bar', data_type_inc='Bar')], - attributes=[AttributeSpec('foo_attr', 'a string attribute specified as text', 'text', - required=False)]) - - return (bar, foo) - - def test_invalid_missing_req_type(self): - foo_builder = GroupBuilder('my_foo', attributes={'data_type': 'Foo', - 'foo_attr': text('example Foo object')}) - results = self.vmap.validate(foo_builder) - self.assertIsInstance(results[0], MissingDataType) # noqa: F405 - self.assertEqual(results[0].name, 'Foo') - self.assertEqual(results[0].reason, 'missing data type Bar') - - def test_invalid_wrong_name_req_type(self): - bar_builder = GroupBuilder('bad_bar_name', - attributes={'data_type': 'Bar', 'attr1': 'a string attribute'}, - datasets=[DatasetBuilder('data', 100, attributes={'attr2': 10})]) - - foo_builder = GroupBuilder('my_foo', - attributes={'data_type': 'Foo', 'foo_attr': text('example Foo object')}, - groups=[bar_builder]) - - results = self.vmap.validate(foo_builder) - self.assertEqual(len(results), 1) - self.assertIsInstance(results[0], MissingDataType) # noqa: F405 - self.assertEqual(results[0].data_type, 'Bar') - - def test_valid(self): - bar_builder = GroupBuilder('my_bar', - attributes={'data_type': 'Bar', 'attr1': text('a string attribute')}, - datasets=[DatasetBuilder('data', 100, attributes={'attr2': 10})]) - - foo_builder = GroupBuilder('my_foo', - attributes={'data_type': 'Foo', 'foo_attr': text('example Foo object')}, - groups=[bar_builder]) - - results = self.vmap.validate(foo_builder) - self.assertEqual(len(results), 0) - - def test_valid_wo_opt_attr(self): - bar_builder = GroupBuilder('my_bar', - attributes={'data_type': 'Bar', 'attr1': text('a string attribute')}, - datasets=[DatasetBuilder('data', 100, attributes={'attr2': 10})]) - foo_builder = GroupBuilder('my_foo', - attributes={'data_type': 'Foo'}, - groups=[bar_builder]) - - results = self.vmap.validate(foo_builder) - self.assertEqual(len(results), 0) diff --git a/tests/unit/pynwb_tests/test_base.py b/tests/unit/pynwb_tests/test_base.py index b66d476d0..38a653611 100644 --- a/tests/unit/pynwb_tests/test_base.py +++ b/tests/unit/pynwb_tests/test_base.py @@ -2,8 +2,8 @@ import numpy as np from pynwb.base import ProcessingModule, TimeSeries, Images, Image -from pynwb.form.data_utils import DataChunkIterator -from pynwb.form.backends.hdf5 import H5DataIO +from hdmf.data_utils import DataChunkIterator +from hdmf.backends.hdf5 import H5DataIO class TestProcessingModule(unittest.TestCase): diff --git a/tests/unit/pynwb_tests/test_extension.py b/tests/unit/pynwb_tests/test_extension.py index bfb5c239f..630f336de 100644 --- a/tests/unit/pynwb_tests/test_extension.py +++ b/tests/unit/pynwb_tests/test_extension.py @@ -7,8 +7,8 @@ import unittest2 as unittest from pynwb import get_type_map, TimeSeries, NWBFile, register_class, docval, load_namespaces, popargs -from pynwb.form.spec.spec import RefSpec -from pynwb.form.utils import get_docval +from hdmf.spec.spec import RefSpec +from hdmf.utils import get_docval from pynwb.spec import NWBNamespaceBuilder, NWBGroupSpec, NWBAttributeSpec, NWBDatasetSpec from pynwb.file import LabMetaData diff --git a/tox.ini b/tox.ini index 7a15a0629..2f49ede77 100644 --- a/tox.ini +++ b/tox.ini @@ -22,7 +22,7 @@ commands = [testenv:localcoverage] basepython = python3.6 commands = - coverage run test.py --form --pynwb + coverage run test.py --pynwb coverage html -d tests/coverage/htmlcov # Envs that builds wheels and source distribution @@ -50,7 +50,7 @@ commands = {[testenv:build]commands} # Envs that will only be executed on CI that does coverage reporting [testenv:coverage] commands = - coverage run test.py --form --pynwb + coverage run test.py --pynwb coverage report -m codecov -X fix