From 75f98c741c38105e5ea239b72a4715b92091e75a Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2020 19:41:04 +0200 Subject: [PATCH 01/29] Code quality: Variable renames --- nixio/dimensions.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 0eda9d22..b7e02ff9 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -133,9 +133,9 @@ def label(self): return self._h5group.get_attr("label") @label.setter - def label(self, l): - util.check_attr_type(l, str) - self._h5group.set_attr("label", l) + def label(self, label): + util.check_attr_type(label, str) + self._h5group.set_attr("label", label) @property def sampling_interval(self): @@ -229,9 +229,9 @@ def label(self): return self._redirgrp.get_attr("label") @label.setter - def label(self, l): - util.check_attr_type(l, str) - self._redirgrp.set_attr("label", l) + def label(self, label): + util.check_attr_type(label, str) + self._redirgrp.set_attr("label", label) @property def unit(self): @@ -311,7 +311,7 @@ def create_new(cls, data_array, index): def labels(self): labels = tuple(self._h5group.get_data("labels")) if len(labels) and isinstance(labels[0], bytes): - labels = tuple(l.decode() for l in labels) + labels = tuple(label.decode() for label in labels) return labels @labels.setter From 6a083f830e9cb703e5a01d643dae7d7aa8b92703 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2020 15:07:40 +0200 Subject: [PATCH 02/29] New DimensionLink class --- nixio/dimensions.py | 105 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index b7e02ff9..9b312eb5 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -8,6 +8,10 @@ # modification, are permitted under the terms of the BSD License. See # LICENSE file in the root of the Project. from numbers import Number +try: + from collections.abc import Sequence +except ImportError: + from collections import Sequence import numpy as np @@ -16,6 +20,7 @@ from .data_frame import DataFrame from . import util from .container import Container +from .exceptions import IncompatibleDimensions, OutOfBounds class DimensionContainer(Container): @@ -35,6 +40,80 @@ def _inst_item(self, item): return cls(self._parent, idx) +class DimensionLink(object): + """ + Links a Dimension to a data object (DataArray or DataFrame). + + A single vector of values from the underlying data object must be + specified to serve as the ticks or labels for the associated Dimension. + + - A single vector of a DataArray. + - A single column of a DataFrame. + """ + + def __init__(self, nixfile, nixparent, h5group): + util.check_entity_id(h5group.get_attr("entity_id")) + self._h5group = h5group + self._parent = nixparent + self._file = nixfile + + @classmethod + def create_new(cls, nixfile, nixparent, h5parent, dataobj, dotype, index): + id_ = util.create_id() + h5group = h5parent.open_group("link", True) + h5group.set_attr("entity_id", id_) + newdimlink = cls(nixfile, nixparent, h5group) + newdimlink._h5group.set_attr("data_object_type", dotype) + newdimlink._h5group.create_link(dataobj, dataobj.id) + newdimlink.index = index + now = util.time_to_str(util.now_int()) + newdimlink._h5group.set_attr("created_at", now) + newdimlink._h5group.set_attr("updated_at", now) + return newdimlink + + @property + def id(self): + self._h5group.get_attr("entity_id") + + @property + def file(self): + return self._file + + @property + def linked_data(self): + grp = self._h5group.get_by_pos(0) + return grp.get_data("data") + + @property + def index(self): + if self._data_object_type == "DataArray": + return tuple(self._h5group.get_attr("index")) + if self._data_object_type == "DataFrame": + return self._h5group.get_attr("index") + raise RuntimeError("Invalid DataObjectType attribute found in " + "DimensionLink") + + @index.setter + def index(self, index): + if self._data_object_type == "DataArray": + util.check_attr_type(index, Sequence) + if index.count(-1) != 1: + raise ValueError("Index for DimensionLink with DataArray must " + "have exactly one value equal to -1. " + "See class docstring for more information.") + self._h5group.set_attr("index", list(index)) + elif self._data_object_type == "DataFrame": + util.check_attr_type(index, int) + self._h5group.set_attr("index", index) + else: + raise RuntimeError("Invalid DataObjectType attribute found in " + "DimensionLink") + + @property + def _data_object_type(self): + return self._h5group.get_attr("data_object_type") + + class Dimension(object): def __init__(self, nixfile, data_array, index): @@ -59,6 +138,32 @@ def _set_dimension_type(self, dimtype): def index(self): return self.dim_index + def link_data_array(self, data_array, index): + if len(data_array.data_extent) != len(index): + raise IncompatibleDimensions( + "Length of linked DataArray indices ({}) does not match" + "number of DataArray dimensions ({})".format( + len(data_array.data_extent), len(index) + ), + "Dimension.link_data_array" + ) + + if index.count(-1) != 1: + # TODO: Add link to relevant docs + raise ValueError( + "Invalid linked DataArray index: " + "One of the values must be -1, indicating the relevant vector." + ) + + DimensionLink.create_new(self._file, self, self._h5group, + data_array, "DataArray", index) + + def link_data_frame(self, data_frame, index): + if index >= len(data_frame.columns): + raise OutOfBounds("DataFrame index is out of bounds", index) + DimensionLink.create_new(self._file, self, self._h5group, + data_frame, "DataFrame", index) + def __str__(self): return "{}: {{index = {}}}".format( type(self).__name__, self.index From 08f38ef7d3fe31c79213956a12ef25252156d11e Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 14 Jul 2020 18:36:59 +0200 Subject: [PATCH 03/29] Allow creating range dimensions without ticks For linking later. --- nixio/data_array.py | 2 +- nixio/dimensions.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nixio/data_array.py b/nixio/data_array.py index 22d1a9a9..75c055a1 100644 --- a/nixio/data_array.py +++ b/nixio/data_array.py @@ -118,7 +118,7 @@ def append_sampled_dimension(self, sampling_interval, label=None, self.force_updated_at() return smpldim - def append_range_dimension(self, ticks, label=None, unit=None): + def append_range_dimension(self, ticks=None, label=None, unit=None): """ Append a new RangeDimension to the list of existing dimension descriptors. diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 9b312eb5..854bb334 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -280,7 +280,8 @@ def __init__(self, data_array, index): def create_new(cls, data_array, index, ticks): newdim = cls(data_array, index) newdim._set_dimension_type(DimensionType.Range) - newdim._h5group.write_data("ticks", ticks, dtype=DataType.Double) + if ticks is not None: + newdim._h5group.write_data("ticks", ticks, dtype=DataType.Double) return newdim @classmethod From ac5534080e2de9f6fc686195ced21917d13025da Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 14 Jul 2020 18:43:15 +0200 Subject: [PATCH 04/29] [LinkDimension] value and unit getters --- nixio/dimensions.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 854bb334..19f11add 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -109,6 +109,39 @@ def index(self, index): raise RuntimeError("Invalid DataObjectType attribute found in " "DimensionLink") + @property + def values(self): + """ + Returns the values (vector or column) from the linked data object + (DataArray or DataFrame) specified by the LinkDimension's index. + """ + data = self.linked_data + if self._data_object_type == "DataArray": + dimindex = list(self.index) + # replace -1 with slice(None) + dimindex[dimindex.index(-1)] = slice(None) + return data[dimindex] + elif self._data_object_type == "DataFrame": + return data[self.index] + else: + raise RuntimeError("Invalid DataObjectType attribute found in " + "DimensionLink") + + @property + def unit(self): + """ + Returns the unit from the linked data object (DataArray or DataFrame) + specified by the LinkDimension's index. + """ + data = self.linked_data + if self._data_object_type == "DataArray": + return data.get_attr("unit") + elif self._data_object_type == "DataFrame": + return data.get_attr("units")[self.index] + else: + raise RuntimeError("Invalid DataObjectType attribute found in " + "DimensionLink") + @property def _data_object_type(self): return self._h5group.get_attr("data_object_type") From d61dbc035d85a6fa922d9fb5e14683fe32561df4 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 14 Jul 2020 19:41:09 +0200 Subject: [PATCH 05/29] Base Dimension methods for accessing linked data --- nixio/dimensions.py | 44 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 19f11add..b9695935 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -197,6 +197,40 @@ def link_data_frame(self, data_frame, index): DimensionLink.create_new(self._file, self, self._h5group, data_frame, "DataFrame", index) + @property + def has_link(self): + """ + Return True if this Dimension links to a data object + (DataArray or DataFrame). + Read-only property. + """ + if "link" in self._h5group: + return True + return False + + @property + def _redirgrp(self): + """ + If the dimension links to a data object, this property returns + the H5Group of the linked DataArray or DataFrame. Otherwise, it returns + the H5Group representing the dimension. + """ + if self.has_link: + link = self._h5group.open_group("link") + return link.get_by_pos(0) + return self._h5group + + @property + def dimension_link(self): + """ + If the dimension has a DimensionLink to a data object, returns the + DimensionLink object, otherwise returns None. + """ + if self.has_link: + link = self._h5group.open_group("link") + return DimensionLink(self._file, self, link) + return None + def __str__(self): return "{}: {{index = {}}}".format( type(self).__name__, self.index @@ -268,12 +302,12 @@ def axis(self, count, start=0): @property def label(self): - return self._h5group.get_attr("label") + return self._redirgrp.get_attr("label") @label.setter def label(self, label): util.check_attr_type(label, str) - self._h5group.set_attr("label", label) + self._redirgrp.set_attr("label", label) @property def sampling_interval(self): @@ -286,12 +320,12 @@ def sampling_interval(self, interval): @property def unit(self): - return self._h5group.get_attr("unit") + return self._redirgrp.get_attr("unit") @unit.setter def unit(self, u): util.check_attr_type(u, str) - self._h5group.set_attr("unit", u) + self._redirgrp.set_attr("unit", u) @property def offset(self): @@ -352,7 +386,7 @@ def ticks(self, ticks): self._h5group.write_data("ticks", ticks) @property - def _redirgrp(self): + def _redirgrp_old(self): """ If the dimension is an Alias Range dimension, this property returns the H5Group of the linked DataArray. Otherwise, it returns the H5Group From 4f7bf5547a176316bda77e971df0749990420d68 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 14 Jul 2020 19:46:03 +0200 Subject: [PATCH 06/29] Disable linking for SampledDimension --- nixio/dimensions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index b9695935..39475bd2 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -336,6 +336,12 @@ def offset(self, o): util.check_attr_type(o, Number) self._h5group.set_attr("offset", o) + def link_data_array(self, data_array, index): + raise RuntimeError("SampledDimension does not support linking") + + def link_data_frame(self, data_array, index): + raise RuntimeError("SampledDimension does not support linking") + class RangeDimension(Dimension): From 3f683e7a47c6bc407d703036e1720025be8897db Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 14 Jul 2020 21:30:34 +0200 Subject: [PATCH 07/29] [DimensionLink] Delete old link before creating new --- nixio/dimensions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 39475bd2..09c0c367 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -188,12 +188,16 @@ def link_data_array(self, data_array, index): "One of the values must be -1, indicating the relevant vector." ) + if self.has_link: + self._h5group.delete("link") DimensionLink.create_new(self._file, self, self._h5group, data_array, "DataArray", index) def link_data_frame(self, data_frame, index): if index >= len(data_frame.columns): raise OutOfBounds("DataFrame index is out of bounds", index) + if self.has_link: + self._h5group.delete("link") DimensionLink.create_new(self._file, self, self._h5group, data_frame, "DataFrame", index) From de4f45515f59e33fb3ed37d1b2dc3f9bc983bfc2 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 14 Jul 2020 21:32:05 +0200 Subject: [PATCH 08/29] Use get_by_name for opening subgroups --- nixio/dimensions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 09c0c367..f44f5f05 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -220,7 +220,7 @@ def _redirgrp(self): the H5Group representing the dimension. """ if self.has_link: - link = self._h5group.open_group("link") + link = self._h5group.get_by_name("link") return link.get_by_pos(0) return self._h5group @@ -231,7 +231,7 @@ def dimension_link(self): DimensionLink object, otherwise returns None. """ if self.has_link: - link = self._h5group.open_group("link") + link = self._h5group.get_by_name("link") return DimensionLink(self._file, self, link) return None From f39082c8fed3a63056acf2de3dd325f13251988a Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 14 Jul 2020 21:33:01 +0200 Subject: [PATCH 09/29] RangeDimension.ticks: Read through to linked data --- nixio/dimensions.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index f44f5f05..e9c9b6ce 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -380,13 +380,10 @@ def is_alias(self): @property def ticks(self): - g = self._redirgrp - if g.has_data("ticks"): - ticks = g.get_data("ticks") - elif g.has_data("data"): - ticks = g.get_data("data") + if self.has_link: + ticks = self.dimension_link.values else: - raise AttributeError("Attribute 'ticks' is not set.") + ticks = self._h5group.get_data("ticks") return tuple(ticks) @ticks.setter From f21ab60c59e47c4353d379a89652c6de6f161afb Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 14 Jul 2020 21:33:33 +0200 Subject: [PATCH 10/29] TestLinkDimension: RangeDimension->DA link tests - Replace alias range dimension test with self-link test. - Test RangeDimension -> DataArray link --- nixio/test/test_dimensions.py | 70 ++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index 0c417d7e..7f6776c0 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -135,7 +135,7 @@ def test_range_dimension(self): self.range_dim.axis(10, 2) self.range_dim.axis(100) - def test_alias_dimension(self): + def _test_alias_dimension(self): da = self.block.create_data_array("alias da", "dimticks", data=np.random.random(10)) da.label = "alias dimension label" @@ -192,7 +192,73 @@ def test_append_dim_init(self): assert sunit == smpldim.unit assert soffset == smpldim.offset - def test_df_dim(self): + +class TestLinkDimension(unittest.TestCase): + + def setUp(self): + self.tmpdir = TempDir("linkdimtest") + self.testfilename = os.path.join(self.tmpdir.path, "linkdimtest.nix") + self.file = nix.File.open(self.testfilename, nix.FileMode.Overwrite) + self.block = self.file.create_block("test block", "test.session") + self.array = self.block.create_data_array( + "test array", "signal", nix.DataType.Float, + data=np.random.random((3, 15)) + ) + + self.set_dim = self.array.append_set_dimension() + self.range_dim = self.array.append_range_dimension() + + def tearDown(self): + del self.file.blocks[self.block.id] + self.file.close() + self.tmpdir.cleanup() + + def test_data_array_range_link_dimension(self): + tickarray = self.block.create_data_array( + "ticks", "array.dimension.ticks", + data=np.linspace(0, 100, 15) + ) + tickarray.label = "DIMENSION LABEL" + tickarray.unit = "mV" + self.range_dim.link_data_array(tickarray, [-1]) + assert np.all(tickarray[:] == self.range_dim.ticks) + assert np.all(tickarray.unit == self.range_dim.unit) + assert np.all(tickarray.label == self.range_dim.label) + + tickarray3d = self.block.create_data_array( + "ticks3d", "array.dimension.ticks", + data=np.random.random((20, 15, 4)) + ) + tickarray3d.unit = "mA" + ticks = np.cumsum(np.random.random(15)) + tickarray3d[3, :, 1] = ticks + tickarray3d.label = "DIMENSION LABEL 2" + self.range_dim.link_data_array(tickarray3d, [3, -1, 1]) + assert np.shape(ticks) == np.shape(self.range_dim.ticks) + assert np.all(ticks == self.range_dim.ticks) + assert np.all(tickarray3d.unit == self.range_dim.unit) + assert np.all(tickarray3d.label == self.range_dim.label) + + def test_data_array_set_link_dimension(self): + pass + + def test_data_array_self_link_dimension(self): + # The new way of making alias range dimension + da = self.block.create_data_array("alias da", "dimticks", + data=np.random.random(10)) + da.label = "alias dimension label" + da.unit = "F" + rdim = da.append_range_dimension() + rdim.link_data_array(da, [-1]) + assert(len(da.dimensions) == 1) + assert(da.dimensions[0].label == da.label) + assert(da.dimensions[0].unit == da.unit) + assert(np.all(da.dimensions[0].ticks == da[:])) + + def test_data_frame_range_link_dimension(self): + pass + + def _test_data_frame_set_link_dimension(self): di = OrderedDict([('name', np.int64), ('id', str), ('time', float), ('sig1', np.float64), ('sig2', np.int32)]) arr = [(1, "a", 20.18, 5.0, 100), (2, 'b', 20.09, 5.5, 101), From c53255fc3121b4bbef3ef743d5cf54b3f4c08092 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 15 Jul 2020 16:00:03 +0200 Subject: [PATCH 11/29] Multidimensional indexing should use a tuple Deprecation warning FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result. --- nixio/dimensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index e9c9b6ce..e4089ba1 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -120,7 +120,7 @@ def values(self): dimindex = list(self.index) # replace -1 with slice(None) dimindex[dimindex.index(-1)] = slice(None) - return data[dimindex] + return data[tuple(dimindex)] elif self._data_object_type == "DataFrame": return data[self.index] else: From f06e8d901aa2743376913396188e37451c8a6f31 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 15 Jul 2020 16:14:01 +0200 Subject: [PATCH 12/29] SetDimension.labels: Read through to linked data --- nixio/dimensions.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index e4089ba1..a8c4c254 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -489,10 +489,15 @@ def create_new(cls, data_array, index): @property def labels(self): - labels = tuple(self._h5group.get_data("labels")) + if self.has_link: + labels = self.dimension_link.values + else: + labels = self._h5group.get_data("labels") + if len(labels) and isinstance(labels[0], bytes): labels = tuple(label.decode() for label in labels) - return labels + + return tuple(labels) @labels.setter def labels(self, labels): From a0063404c33a92cd528d24f4413243e15bd21ffd Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 15 Jul 2020 16:15:10 +0200 Subject: [PATCH 13/29] TestLinkDimension: SetDimension->DA link tests --- nixio/test/test_dimensions.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index 7f6776c0..31138b10 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -240,7 +240,22 @@ def test_data_array_range_link_dimension(self): assert np.all(tickarray3d.label == self.range_dim.label) def test_data_array_set_link_dimension(self): - pass + labelarray = self.block.create_data_array( + "labels", "array.dimension.labels", + data=["Alpha", "Beta", "Gamma"], dtype=nix.DataType.String + ) + self.set_dim.link_data_array(labelarray, [-1]) + assert np.all(labelarray[:] == self.set_dim.labels) + + labelarray2d = self.block.create_data_array( + "labels2d", "array.dimension.labels", + dtype=nix.DataType.String, + data=[["Alpha1", "Beta1", "Gamma1"], + ["Alpha2", "Beta2", "Gamma2"]], + ) + self.set_dim.link_data_array(labelarray2d, [1, -1]) + assert np.all(("Alpha2", "Beta2", "Gamma2") == self.set_dim.labels) + assert np.all(labelarray2d[1, :] == self.set_dim.labels) def test_data_array_self_link_dimension(self): # The new way of making alias range dimension From c73bb77efbedcbc9f50deb76f14b216ad8452061 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 15 Jul 2020 16:20:44 +0200 Subject: [PATCH 14/29] TestLinkDimension: Self link tests For both Range and Set dimensions --- nixio/test/test_dimensions.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index 31138b10..e557a08f 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -257,7 +257,7 @@ def test_data_array_set_link_dimension(self): assert np.all(("Alpha2", "Beta2", "Gamma2") == self.set_dim.labels) assert np.all(labelarray2d[1, :] == self.set_dim.labels) - def test_data_array_self_link_dimension(self): + def test_data_array_self_link_range_dimension(self): # The new way of making alias range dimension da = self.block.create_data_array("alias da", "dimticks", data=np.random.random(10)) @@ -265,10 +265,19 @@ def test_data_array_self_link_dimension(self): da.unit = "F" rdim = da.append_range_dimension() rdim.link_data_array(da, [-1]) - assert(len(da.dimensions) == 1) - assert(da.dimensions[0].label == da.label) - assert(da.dimensions[0].unit == da.unit) - assert(np.all(da.dimensions[0].ticks == da[:])) + assert len(da.dimensions) == 1 + assert da.dimensions[0].label == da.label + assert da.dimensions[0].unit == da.unit + assert np.all(da.dimensions[0].ticks == da[:]) + + def test_data_array_self_link_set_dimension(self): + # The new way of making alias range dimension + labelda = self.block.create_data_array("alias da", "dimlabels", + data=np.random.random(10)) + rdim = labelda.append_set_dimension() + rdim.link_data_array(labelda, [-1]) + assert len(labelda.dimensions) == 1 + assert np.all(labelda.dimensions[0].labels == labelda[:]) def test_data_frame_range_link_dimension(self): pass From b59c7ace3aabc0fd6aea0f722327d252b6a1aa01 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 15 Jul 2020 17:06:20 +0200 Subject: [PATCH 15/29] DimensionLink to DataFrame: Apply index to column --- nixio/dimensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index a8c4c254..5650777e 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -122,7 +122,7 @@ def values(self): dimindex[dimindex.index(-1)] = slice(None) return data[tuple(dimindex)] elif self._data_object_type == "DataFrame": - return data[self.index] + return tuple(row[self.index] for row in data) else: raise RuntimeError("Invalid DataObjectType attribute found in " "DimensionLink") From 86b807fc928c305986fdf5dc0a68a3c1129ef057 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 15 Jul 2020 17:38:33 +0200 Subject: [PATCH 16/29] Remove unnecessary SampledDimension redirect Using group redirect method is unnecessary for SampledDimensions since they don't support linking --- nixio/dimensions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 5650777e..0ca9449a 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -306,12 +306,12 @@ def axis(self, count, start=0): @property def label(self): - return self._redirgrp.get_attr("label") + return self._h5group.get_attr("label") @label.setter def label(self, label): util.check_attr_type(label, str) - self._redirgrp.set_attr("label", label) + self._h5group.set_attr("label", label) @property def sampling_interval(self): @@ -324,12 +324,12 @@ def sampling_interval(self, interval): @property def unit(self): - return self._redirgrp.get_attr("unit") + return self._h5group.get_attr("unit") @unit.setter def unit(self, u): util.check_attr_type(u, str) - self._redirgrp.set_attr("unit", u) + self._h5group.set_attr("unit", u) @property def offset(self): From a5d7be46c5c3141e074827e2d2e1e3b2d339c7c6 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 15 Jul 2020 17:43:01 +0200 Subject: [PATCH 17/29] RangeDimension.unit: Read through to linked unit Return unit directly for DataArray links. Return unit of indexed column for DataFrame links. --- nixio/dimensions.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 0ca9449a..606b279b 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -79,9 +79,12 @@ def id(self): def file(self): return self._file + def _linked_group(self): + return self._h5group.get_by_pos(0) + @property def linked_data(self): - grp = self._h5group.get_by_pos(0) + grp = self._linked_group() return grp.get_data("data") @property @@ -133,11 +136,11 @@ def unit(self): Returns the unit from the linked data object (DataArray or DataFrame) specified by the LinkDimension's index. """ - data = self.linked_data + lobj = self._linked_group() if self._data_object_type == "DataArray": - return data.get_attr("unit") + return lobj.get_attr("unit") elif self._data_object_type == "DataFrame": - return data.get_attr("units")[self.index] + return lobj.get_attr("units")[self.index] else: raise RuntimeError("Invalid DataObjectType attribute found in " "DimensionLink") @@ -415,7 +418,9 @@ def label(self, label): @property def unit(self): - return self._redirgrp.get_attr("unit") + if self.has_link: + return self.dimension_link.unit + return self._h5group.get_attr("unit") @unit.setter def unit(self, u): From 55a28c680c5b443066ea18d8afd7bd92249ba9e2 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 15 Jul 2020 17:44:45 +0200 Subject: [PATCH 18/29] RangeDimension.label: Read through to linked label Return label directly for DataArray links. Return name of indexed column for DataFrame links. --- nixio/dimensions.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 606b279b..8e50ea7b 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -145,6 +145,24 @@ def unit(self): raise RuntimeError("Invalid DataObjectType attribute found in " "DimensionLink") + @property + def label(self): + """ + Returns the label of the linked data object: + For DataArray links, returns the label. + For DataFrame links, returns the name of the column specified by the + LinkDimension's index. + """ + lobj = self._linked_group() + if self._data_object_type == "DataArray": + return lobj.get_attr("label") + elif self._data_object_type == "DataFrame": + col_dts = lobj.group["data"].dtype + return col_dts.names[self.index] + else: + raise RuntimeError("Invalid DataObjectType attribute found in " + "DimensionLink") + @property def _data_object_type(self): return self._h5group.get_attr("data_object_type") @@ -409,7 +427,9 @@ def _redirgrp_old(self): @property def label(self): - return self._redirgrp.get_attr("label") + if self.has_link: + return self.dimension_link.label + return self._h5group.get_attr("label") @label.setter def label(self, label): From 8c8f1c0c52a1e87e49b3ea127dad0556eabf8beb Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 15 Jul 2020 17:47:07 +0200 Subject: [PATCH 19/29] TestLinkDimension: DataFrame for Range and Set dimensions --- nixio/test/test_dimensions.py | 98 ++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 36 deletions(-) diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index e557a08f..e77f2cfb 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -280,40 +280,66 @@ def test_data_array_self_link_set_dimension(self): assert np.all(labelda.dimensions[0].labels == labelda[:]) def test_data_frame_range_link_dimension(self): + column_descriptions = OrderedDict([("name", nix.DataType.String), + ("id", nix.DataType.String), + ("duration", nix.DataType.Double)]) + + def randtick(): + ts = 0 + while True: + ts += np.random.random() + yield ts + + tickgen = randtick() + + values = [("Alpha", "a", next(tickgen)), + ("Beta", 'b', next(tickgen)), + ("Gamma", 'c', next(tickgen)), + ("Alpha", "a", next(tickgen)), + ("Gamma", 'c', next(tickgen)), + ("Alpha", "a", next(tickgen)), + ("Gamma", 'c', next(tickgen)), + ("Alpha", "a", next(tickgen)), + ("Beta", 'b', next(tickgen))] + units = (None, None, "s") + df = self.block.create_data_frame("df-dimension", + "array.dimension.labels", + col_dict=column_descriptions, + data=values) + df.units = units + + self.range_dim.link_data_frame(df, 2) + np.testing.assert_almost_equal(self.range_dim.ticks, + tuple(v[2] for v in values)) + assert self.range_dim.unit == df.units[2] + assert self.range_dim.label == df.column_names[2] + + def test_data_frame_set_link_dimension(self): + column_descriptions = OrderedDict([("name", nix.DataType.String), + ("id", nix.DataType.String), + ("duration", nix.DataType.Float)]) + + def rdura(): + return np.random.random() * 30 + + values = [("Alpha", "a", rdura()), + ("Beta", 'b', rdura()), + ("Gamma", 'c', rdura()), + ("Alpha", "a", rdura()), + ("Gamma", 'c', rdura()), + ("Alpha", "a", rdura()), + ("Gamma", 'c', rdura()), + ("Alpha", "a", rdura()), + ("Beta", 'b', rdura())] + units = (None, None, "s") + df = self.block.create_data_frame("df-dimension", + "array.dimension.labels", + col_dict=column_descriptions, + data=values) + df.units = units + + self.set_dim.link_data_frame(df, 1) + assert self.set_dim.labels == tuple(v[1] for v in values) + + def test_linking_errors(self): pass - - def _test_data_frame_set_link_dimension(self): - di = OrderedDict([('name', np.int64), ('id', str), ('time', float), - ('sig1', np.float64), ('sig2', np.int32)]) - arr = [(1, "a", 20.18, 5.0, 100), (2, 'b', 20.09, 5.5, 101), - (2, 'c', 20.05, 5.1, 100), (1, "d", 20.15, 5.3, 150), - (2, 'e', 20.23, 5.7, 200), (2, 'f', 20.07, 5.2, 300), - (1, "g", 20.12, 5.1, 39), (1, "h", 20.27, 5.1, 600), - (2, 'i', 20.15, 5.6, 400), (2, 'j', 20.08, 5.1, 200)] - unit = [None, None, "s", "mV", None] - df = self.block.create_data_frame("ref frame", "test", - col_dict=di, data=arr) - df.units = unit - dfdim1 = self.array.append_data_frame_dimension(df) - dfdim2 = self.array.append_data_frame_dimension(df, column_idx=1) - self.assertRaises(ValueError, dfdim1.get_ticks) - for ti, tu in enumerate(arr): - for idx, item in enumerate(tu): - # ticks - assert item == dfdim1.get_ticks(idx)[ti] - assert item == dfdim2.get_ticks(idx)[ti] - assert self.array.dimensions[3].get_ticks(idx)[ti] \ - == dfdim1.get_ticks(idx)[ti] - assert self.array.dimensions[4].get_ticks(idx)[ti] \ - == dfdim2.get_ticks(idx)[ti] - # units - assert unit[idx] == dfdim1.get_unit(idx) - assert unit[idx] == dfdim2.get_unit(idx) - # labels - assert list(di)[idx] == dfdim1.get_label(idx) - assert list(di)[idx] == dfdim2.get_label(idx) - for ti, tu in enumerate(arr): - assert arr[ti][1] == dfdim2.get_ticks()[ti] - assert unit[1] == dfdim2.get_unit() - assert list(di)[1] == dfdim2.get_label() - assert dfdim1.get_label() == df.name From 44f005b81ce4e24834634a7954878380115e2954 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 15 Jul 2020 17:56:22 +0200 Subject: [PATCH 20/29] Remove AliasRangeDimension and DataFrameDimension - Removed DataFrameDimension class. - Removed all mentions of alias range dimension from DataArray. - Removed associated tests and test sections. --- nixio/data_array.py | 67 +-------------- nixio/dimensions.py | 156 +--------------------------------- nixio/test/test_data_array.py | 67 --------------- nixio/test/test_dimensions.py | 11 --- nixio/validator.py | 11 --- 5 files changed, 4 insertions(+), 308 deletions(-) diff --git a/nixio/data_array.py b/nixio/data_array.py index 75c055a1..5aa56398 100644 --- a/nixio/data_array.py +++ b/nixio/data_array.py @@ -16,8 +16,7 @@ from .source_link_container import SourceLinkContainer from .datatype import DataType from .dimensions import (Dimension, SampledDimension, RangeDimension, - SetDimension, DataFrameDimension, - DimensionType, DimensionContainer) + SetDimension, DimensionType, DimensionContainer) from . import util from .compression import Compression @@ -138,61 +137,6 @@ def append_range_dimension(self, ticks=None, label=None, unit=None): self.force_updated_at() return rdim - def append_data_frame_dimension(self, data_frame, column_idx=None): - """ - Append a new DataFrameDimension to the list of existing dimension - descriptors. - - :param data_frame: The referenced DataFrame - :type data_frame: nix.DataFrame - - :param column_idx: Index of the referenced column of the DataFrame. - The default column determines the default label, - ticks, and unit of this Dimension. - :type column_idx: int or None - - :returns: Thew newly created DataFrameDimension. - :rtype: DataFrameDimension - """ - index = len(self.dimensions) + 1 - dfdim = DataFrameDimension.create_new(self, index, - data_frame, column_idx) - if self.file.auto_update_timestamps: - self.force_updated_at() - return dfdim - - def append_alias_range_dimension(self): - """ - Append a new RangeDimension that uses the data stored in this - DataArray as ticks. This works only(!) if the DataArray is 1-D and - the stored data is numeric. A ValueError will be raised otherwise. - - :returns: The created dimension descriptor. - :rtype: RangeDimension - - """ - if (len(self.data_extent) > 1 or - not DataType.is_numeric_dtype(self.dtype)): - raise ValueError("AliasRangeDimensions only allowed for 1D " - "numeric DataArrays.") - if self._dimension_count() > 0: - raise ValueError("Cannot append additional alias dimension. " - "There must only be one!") - # check if existing unit is SI - if self.unit: - u = self.unit - if not (util.units.is_si(u) or util.units.is_compound(u)): - raise InvalidUnit( - "AliasRangeDimensions are only allowed when SI or " - "composites of SI units are used. " - "Current SI unit is {}".format(u), - "DataArray.append_alias_range_dimension" - ) - aliasdim = RangeDimension.create_new_alias(self, 1) - if self.file.auto_update_timestamps: - self.force_updated_at() - return aliasdim - def delete_dimensions(self): """ Delete all the dimension descriptors for this DataArray. @@ -311,15 +255,6 @@ def unit(self, u): if u == "": u = None util.check_attr_type(u, str) - if (self._dimension_count() == 1 and - self.dimensions[0].dimension_type == DimensionType.Range and - self.dimensions[0].is_alias and u is not None): - if not (util.units.is_si(u) or util.units.is_compound(u)): - raise InvalidUnit( - "[{}]: Non-SI units are not allowed if the DataArray " - "has an AliasRangeDimension.".format(u), - "DataArray.unit" - ) self._h5group.set_attr("unit", u) if self.file.auto_update_timestamps: self.force_updated_at() diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 8e50ea7b..85e12702 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -17,7 +17,6 @@ from .datatype import DataType from .dimension_type import DimensionType -from .data_frame import DataFrame from . import util from .container import Container from .exceptions import IncompatibleDimensions, OutOfBounds @@ -34,7 +33,6 @@ def _inst_item(self, item): DimensionType.Range: RangeDimension, DimensionType.Sample: SampledDimension, DimensionType.Set: SetDimension, - DimensionType.DataFrame: DataFrameDimension, }[DimensionType(item.get_attr("dimension_type"))] idx = item.name return cls(self._parent, idx) @@ -382,23 +380,6 @@ def create_new(cls, data_array, index, ticks): newdim._h5group.write_data("ticks", ticks, dtype=DataType.Double) return newdim - @classmethod - def create_new_alias(cls, data_array, index): - newdim = cls(data_array, index) - newdim._set_dimension_type(DimensionType.Range) - newdim._h5group.create_link(data_array, data_array.id) - return newdim - - @property - def is_alias(self): - """ - Return True if this dimension is an Alias Range dimension. - Read-only property. - """ - if self._h5group.has_data("ticks"): - return False - return True - @property def ticks(self): if self.has_link: @@ -409,22 +390,11 @@ def ticks(self): @ticks.setter def ticks(self, ticks): + # TODO: Write through to linked object if np.any(np.diff(ticks) < 0): raise ValueError("Ticks are not given in an ascending order.") self._h5group.write_data("ticks", ticks) - @property - def _redirgrp_old(self): - """ - If the dimension is an Alias Range dimension, this property returns - the H5Group of the linked DataArray. Otherwise, it returns the H5Group - representing the dimension. - """ - if self.is_alias: - gname = self._h5group.get_by_pos(0).name - return self._h5group.open_group(gname) - return self._h5group - @property def label(self): if self.has_link: @@ -433,6 +403,7 @@ def label(self): @label.setter def label(self, label): + # TODO: Write through to linked object util.check_attr_type(label, str) self._redirgrp.set_attr("label", label) @@ -526,127 +497,6 @@ def labels(self): @labels.setter def labels(self, labels): + # TODO: Write through to linked object dt = util.vlen_str_dtype self._h5group.write_data("labels", labels, dtype=dt) - - -class DataFrameDimension(Dimension): - - def __init__(self, data_array, index): - nixfile = data_array.file - super(DataFrameDimension, self).__init__(nixfile, data_array, index) - - @classmethod - def create_new(cls, data_array, index, data_frame, column): - """ - Create a new Dimension that points to a DataFrame - - :param data_array: The DataArray this Dimension belongs to - - :param parent: The H5Group for the dimensions - - :param data_frame: the referenced DataFrame for this Dimension - - :param column: the index of a column in the DataFrame that the - Dimension will reference (optional) - - :return: The new DataFrameDimension - """ - newdim = cls(data_array, index) - newdim.data_frame = data_frame - newdim.column_idx = column - newdim._set_dimension_type(DimensionType.DataFrame) - return newdim - - def get_unit(self, index=None): - """ - Get the unit of the Dimension. If an index is specified, - it will return the unit of the column in the referenced DataFrame at - that index. - - :param index: Index of the needed column - :type index: int - - :return: Unit of the specified column - :rtype: str or None - """ - if index is None: - if self.column_idx is None: - raise ValueError("No default column index is set " - "for this Dimension. Please supply one") - else: - idx = self.column_idx - else: - idx = index - unit = None - if self.data_frame.units is not None: - - unit = self.data_frame.units[idx] - return unit - - def get_ticks(self, index=None): - """ - Get the ticks of the Dimension from the referenced DataFrame. - If an index is specified, it will return the values of the column - in the referenced DataFrame at that index. - - :param index: Index of the needed column - :type index: int - - :return: values in the specified column - :rtype: numpy.ndarray - """ - if index is None: - if self.column_idx is None: - raise ValueError("No default column index is set " - "for this Dimension. Please supply one") - else: - idx = self.column_idx - else: - idx = index - df = self.data_frame - ticks = df[df.column_names[idx]] - return ticks - - def get_label(self, index=None): - """ - Get the label of the Dimension. If an index is specified, - it will return the name of the column in the referenced - DataFrame at that index. - :param index: Index of the referred column - :type index: int or None - - :return: the header of the specified column or the name of DataFrame - if index is None - :rtype: str - """ - if index is None: - if self.column_idx is None: - label = self.data_frame.name - else: - label = self.data_frame.column_names[self.column_idx] - else: - label = self.data_frame.column_names[index] - return label - - @property - def data_frame(self): - dfname = self._h5group.get_by_pos(0).name - grp = self._h5group.open_group(dfname) - nixblock = self._parent._parent - nixfile = self._file - df = DataFrame(nixfile, nixblock, grp) - return df - - @data_frame.setter - def data_frame(self, df): - self._h5group.create_link(df, "data_frame") - - @property - def column_idx(self): - colidx = self._h5group.get_attr("column_idx") - return colidx - - @column_idx.setter - def column_idx(self, col): - self._h5group.set_attr("column_idx", col) diff --git a/nixio/test/test_data_array.py b/nixio/test/test_data_array.py index 49fe0b54..e3370f60 100644 --- a/nixio/test/test_data_array.py +++ b/nixio/test/test_data_array.py @@ -311,43 +311,6 @@ def test_data_array_dimensions(self): self.array.delete_dimensions() - assert(len(self.array.dimensions) == 0) - self.array.append_alias_range_dimension() - assert(len(self.array.dimensions) == 1) - self.array.delete_dimensions() - self.array.append_alias_range_dimension() - assert(len(self.array.dimensions) == 1) - - self.assertRaises(ValueError, self.array.append_alias_range_dimension) - self.assertRaises(ValueError, self.array.append_alias_range_dimension) - string_array = self.block.create_data_array('string_array', - 'nix.texts', - dtype=nix.DataType.String, - shape=(10,)) - self.assertRaises(ValueError, - string_array.append_alias_range_dimension) - assert(len(string_array.dimensions) == 0) - del self.block.data_arrays['string_array'] - - array_2D = self.block.create_data_array( - 'array_2d', 'nix.2d', dtype=nix.DataType.Double, shape=(10, 10) - ) - self.assertRaises(ValueError, array_2D.append_alias_range_dimension) - assert(len(array_2D.dimensions) == 0) - del self.block.data_arrays['array_2d'] - - # alias range dimension with non-SI unit - self.array.delete_dimensions() - self.array.unit = "10 * ms" - with self.assertRaises(ValueError): - self.array.append_alias_range_dimension() - - self.array.delete_dimensions() - self.array.unit = None - self.array.append_alias_range_dimension() - with self.assertRaises(ValueError): - self.array.unit = "10 * ms" - def test_data_array_sources(self): source1 = self.block.create_source("source1", "channel") source2 = self.block.create_source("source2", "electrode") @@ -497,21 +460,6 @@ def test_timestamp_autoupdate(self): array.append_range_dimension(ticks=[0.1]) self.assertNotEqual(datime, array.updated_at) - datime = array.updated_at - time.sleep(1) - df = self.block.create_data_frame( - "df", "test.data_array.timestamp.data_frame", - col_dict={"idx": int} - ) - array.append_data_frame_dimension(data_frame=df) - self.assertNotEqual(datime, array.updated_at) - - array.delete_dimensions() - datime = array.updated_at - time.sleep(1) - array.append_alias_range_dimension() - self.assertNotEqual(datime, array.updated_at) - # other properties datime = array.updated_at time.sleep(1) @@ -553,21 +501,6 @@ def test_timestamp_noautoupdate(self): array.append_range_dimension(ticks=[0.1]) self.assertEqual(datime, array.updated_at) - datime = array.updated_at - time.sleep(1) - df = self.block.create_data_frame( - "df", "test.data_array.timestamp.data_frame", - col_dict={"idx": int} - ) - array.append_data_frame_dimension(data_frame=df) - self.assertEqual(datime, array.updated_at) - - array.delete_dimensions() - datime = array.updated_at - time.sleep(1) - array.append_alias_range_dimension() - self.assertEqual(datime, array.updated_at) - # other properties datime = array.updated_at time.sleep(1) diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index e77f2cfb..2740165b 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -135,17 +135,6 @@ def test_range_dimension(self): self.range_dim.axis(10, 2) self.range_dim.axis(100) - def _test_alias_dimension(self): - da = self.block.create_data_array("alias da", "dimticks", - data=np.random.random(10)) - da.label = "alias dimension label" - da.unit = "F" - da.append_alias_range_dimension() - assert(len(da.dimensions) == 1) - assert(da.dimensions[0].label == da.label) - assert(da.dimensions[0].unit == da.unit) - assert(np.all(da.dimensions[0].ticks == da[:])) - def test_set_dim_label_resize(self): setdim = self.array.append_set_dimension() labels = ["A", "B"] diff --git a/nixio/validator.py b/nixio/validator.py index 1f2d8062..b8f71761 100644 --- a/nixio/validator.py +++ b/nixio/validator.py @@ -462,17 +462,6 @@ def check_sampled_dimension(dim, idx): return errors, warnings -def check_df_dimension(dim, idx): - """ - Validate a DataFrameDimension and return all errors and warnings. - - :returns: A list of 'errors' and a list of 'warnings' - """ - errors = [ValidationError.DataFrameNotMatch.format(idx)] - warnings = list() - return errors, warnings - - def check_entity(entity): """ General NIX entity validator From 3a884081a44bce0c79147b1912ba4d11b10ed4bc Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 15 Jul 2020 19:15:58 +0200 Subject: [PATCH 21/29] Index validation for DataFrame dimension link Should not be negative either. --- nixio/dimensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 85e12702..1d80564f 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -213,7 +213,7 @@ def link_data_array(self, data_array, index): data_array, "DataArray", index) def link_data_frame(self, data_frame, index): - if index >= len(data_frame.columns): + if not 0 <= index < len(data_frame.columns): raise OutOfBounds("DataFrame index is out of bounds", index) if self.has_link: self._h5group.delete("link") From e3a0330c927f4f85594d3b855f3b0691cbf8d7e0 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 15 Jul 2020 19:24:17 +0200 Subject: [PATCH 22/29] Index validation for DataArray dimension link Should not have negative values except one -1. --- nixio/dimensions.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 1d80564f..d55a639f 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -200,12 +200,14 @@ def link_data_array(self, data_array, index): "Dimension.link_data_array" ) - if index.count(-1) != 1: + invalid_idx_msg = ( + "Invalid linked DataArray index: " + "One of the values must be -1, indicating the relevant vector. " + "Negative indexing is not supported." + ) + if index.count(-1) != 1 or sum(idx < 0 for idx in index) != 1: # TODO: Add link to relevant docs - raise ValueError( - "Invalid linked DataArray index: " - "One of the values must be -1, indicating the relevant vector." - ) + raise ValueError(invalid_idx_msg) if self.has_link: self._h5group.delete("link") From 2fd8031008719794ed2e152d4ee23e9a4fa93e34 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 15 Jul 2020 19:26:51 +0200 Subject: [PATCH 23/29] Test dimension link index errors --- nixio/test/test_dimensions.py | 82 ++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 2 deletions(-) diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index 2740165b..cf30f430 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -330,5 +330,83 @@ def rdura(): self.set_dim.link_data_frame(df, 1) assert self.set_dim.labels == tuple(v[1] for v in values) - def test_linking_errors(self): - pass + def test_data_array_linking_errors(self): + da = self.block.create_data_array("baddim", "dimension.error.test", + data=np.random.random((3, 4, 2))) + + # index dimensionality mismatch + with self.assertRaises(nix.exceptions.IncompatibleDimensions): + self.set_dim.link_data_array(da, [0, -1]) + with self.assertRaises(nix.exceptions.IncompatibleDimensions): + self.range_dim.link_data_array(da, [0, -1, 3, 3]) + + # no -1 in index + with self.assertRaises(ValueError): + self.set_dim.link_data_array(da, [0, 0, 0]) + with self.assertRaises(ValueError): + self.range_dim.link_data_array(da, [1, 1, 1]) + + # multiple -1 (or negative)in index + with self.assertRaises(ValueError): + self.set_dim.link_data_array(da, [-1, -1, 0]) + with self.assertRaises(ValueError): + self.range_dim.link_data_array(da, [-1, -1, 1]) + with self.assertRaises(ValueError): + self.set_dim.link_data_array(da, [-2, 0, 0]) + with self.assertRaises(ValueError): + self.range_dim.link_data_array(da, [-2, 0, 1]) + with self.assertRaises(ValueError): + self.set_dim.link_data_array(da, [-10, -10, 0]) + with self.assertRaises(ValueError): + self.range_dim.link_data_array(da, [-10, -10, 1]) + + def test_data_frame_linking_errors(self): + column_descriptions = OrderedDict([("name", nix.DataType.String), + ("id", nix.DataType.String), + ("score", nix.DataType.Float)]) + values = [("Alpha", "a", 0), + ("Beta", 'b', 100), + ("Gamma", 'c', 50), + ("Alpha", "a", 10), + ("Gamma", 'c', 42), + ("Alpha", "a", 93), + ("Gamma", 'c', 23), + ("Alpha", "a", 37), + ("Beta", 'b', 87)] + df = self.block.create_data_frame("df-dimension", + "array.dimension.labels", + col_dict=column_descriptions, + data=values) + for badidx in [-110, -5, -1, 3, 4, 34]: + with self.assertRaises(nix.exceptions.OutOfBounds): + self.set_dim.link_data_frame(df, badidx) + with self.assertRaises(nix.exceptions.OutOfBounds): + self.range_dim.link_data_frame(df, badidx) + + def test_sampled_dimension_unsupported(self): + sdim = self.array.append_sampled_dimension(0.1) + + da = self.block.create_data_array("baddim", "dimension.error.test", + data=np.random.random((3, 4, 2))) + with self.assertRaises(RuntimeError): + sdim.link_data_array(da, [0]) + + column_descriptions = OrderedDict([("name", nix.DataType.String), + ("id", nix.DataType.String), + ("duration", nix.DataType.Float)]) + values = [("Alpha", "a", 0), + ("Beta", 'b', 0), + ("Gamma", 'c', 0), + ("Alpha", "a", 0), + ("Gamma", 'c', 0), + ("Alpha", "a", 0), + ("Gamma", 'c', 0), + ("Alpha", "a", 0), + ("Beta", 'b', 0)] + df = self.block.create_data_frame("df-dimension", + "array.dimension.labels", + col_dict=column_descriptions, + data=values) + + with self.assertRaises(RuntimeError): + sdim.link_data_array(df, 10) From 47cb7536f159259f20a797d3ec4b9fb479359f8e Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 16 Jul 2020 21:32:21 +0200 Subject: [PATCH 24/29] TestLinkDimension: Remove unnecessary np.app() checks --- nixio/test/test_dimensions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index cf30f430..b58126d7 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -211,8 +211,8 @@ def test_data_array_range_link_dimension(self): tickarray.unit = "mV" self.range_dim.link_data_array(tickarray, [-1]) assert np.all(tickarray[:] == self.range_dim.ticks) - assert np.all(tickarray.unit == self.range_dim.unit) - assert np.all(tickarray.label == self.range_dim.label) + assert tickarray.unit == self.range_dim.unit + assert tickarray.label == self.range_dim.label tickarray3d = self.block.create_data_array( "ticks3d", "array.dimension.ticks", @@ -225,8 +225,8 @@ def test_data_array_range_link_dimension(self): self.range_dim.link_data_array(tickarray3d, [3, -1, 1]) assert np.shape(ticks) == np.shape(self.range_dim.ticks) assert np.all(ticks == self.range_dim.ticks) - assert np.all(tickarray3d.unit == self.range_dim.unit) - assert np.all(tickarray3d.label == self.range_dim.label) + assert tickarray3d.unit == self.range_dim.unit + assert tickarray3d.label == self.range_dim.label def test_data_array_set_link_dimension(self): labelarray = self.block.create_data_array( From ecc25cd1abac82c24bd8e522e84e3f91c38c0d2d Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 16 Jul 2020 22:18:52 +0200 Subject: [PATCH 25/29] DimensionLink: Write through label and unit Can now write through to the linked data object when setting label and unit on RangeDimension: - RangeDimension ticks cannot be changed. - RangeDimension label linked to DataArray changes the DataArray label. - RangeDimension label linked to DataFrame raises error: The column name of a DataFrame can't be changed since it's the name of a field of a compound data type, which we can't change. - RangeDimension unit linked to DataArray changes the DataArray unit. - RangeDimension unit linked to DataFrame changes the DataFrame column unit. - SetDimension labels cannot be changed. --- nixio/dimensions.py | 62 ++++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index d55a639f..76837f11 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -143,6 +143,22 @@ def unit(self): raise RuntimeError("Invalid DataObjectType attribute found in " "DimensionLink") + @unit.setter + def unit(self, unit): + """ + Sets the unit of the linked data object. + """ + lobj = self._linked_group() + if self._data_object_type == "DataArray": + lobj.set_attr("unit", unit) + elif self._data_object_type == "DataFrame": + units = list(lobj.get_attr("units")) + units[self.index] = unit + lobj.set_attr("units", units) + else: + raise RuntimeError("Invalid DataObjectType attribute found in " + "DimensionLink") + @property def label(self): """ @@ -161,6 +177,21 @@ def label(self): raise RuntimeError("Invalid DataObjectType attribute found in " "DimensionLink") + @label.setter + def label(self, label): + """ + Sets the label of the linked data objet. + """ + lobj = self._linked_group() + if self._data_object_type == "DataArray": + lobj.set_attr("label", label) + elif self._data_object_type == "DataFrame": + raise RuntimeError("The label of a Dimension linked to a " + "DataFrame column cannot be modified") + else: + raise RuntimeError("Invalid DataObjectType attribute found in " + "DimensionLink") + @property def _data_object_type(self): return self._h5group.get_attr("data_object_type") @@ -233,18 +264,6 @@ def has_link(self): return True return False - @property - def _redirgrp(self): - """ - If the dimension links to a data object, this property returns - the H5Group of the linked DataArray or DataFrame. Otherwise, it returns - the H5Group representing the dimension. - """ - if self.has_link: - link = self._h5group.get_by_name("link") - return link.get_by_pos(0) - return self._h5group - @property def dimension_link(self): """ @@ -392,9 +411,11 @@ def ticks(self): @ticks.setter def ticks(self, ticks): - # TODO: Write through to linked object if np.any(np.diff(ticks) < 0): raise ValueError("Ticks are not given in an ascending order.") + if self.has_link: + raise RuntimeError("The ticks of a RangeDimension linked to a " + "data object cannot be modified") self._h5group.write_data("ticks", ticks) @property @@ -405,9 +426,11 @@ def label(self): @label.setter def label(self, label): - # TODO: Write through to linked object util.check_attr_type(label, str) - self._redirgrp.set_attr("label", label) + if self.has_link: + self.dimension_link.label = label + else: + self._h5group.set_attr("label", label) @property def unit(self): @@ -418,7 +441,10 @@ def unit(self): @unit.setter def unit(self, u): util.check_attr_type(u, str) - self._redirgrp.set_attr("unit", u) + if self.has_link: + self.dimension_link.unit = u + else: + self._h5group.set_attr("unit", u) def index_of(self, position): """ @@ -499,6 +525,8 @@ def labels(self): @labels.setter def labels(self, labels): - # TODO: Write through to linked object + if self.has_link: + raise RuntimeError("The labels of a SetDimension linked to a " + "data object cannot be modified") dt = util.vlen_str_dtype self._h5group.write_data("labels", labels, dtype=dt) From c01021cd36318c44dcd4e85eafca041884634b95 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 16 Jul 2020 22:22:07 +0200 Subject: [PATCH 26/29] TestLinkDimension: Property change tests --- nixio/test/test_dimensions.py | 54 +++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index b58126d7..c136fba9 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -410,3 +410,57 @@ def test_sampled_dimension_unsupported(self): with self.assertRaises(RuntimeError): sdim.link_data_array(df, 10) + + def test_write_linked_array_props(self): + tickarray = self.block.create_data_array( + "ticks3d", "array.dimension.ticks", + data=np.random.random((10, 5, 4)) + ) + tickarray.unit = "whatever" + ticks = np.cumsum(np.random.random(5)) + tickarray[3, :, 1] = ticks + tickarray.label = "DIMENSION LABEL" + self.range_dim.link_data_array(tickarray, [3, -1, 1]) + assert tickarray.unit == self.range_dim.unit + assert tickarray.label == self.range_dim.label + + self.range_dim.unit = "something else" + assert tickarray.unit == "something else" + assert tickarray.unit == self.range_dim.unit + + self.range_dim.label = "MODIFIED DIMENSION LABEL" + assert tickarray.label == "MODIFIED DIMENSION LABEL" + assert tickarray.unit == self.range_dim.unit + + def test_write_linked_dataframe_props(self): + column_descriptions = OrderedDict([("name", nix.DataType.String), + ("id", nix.DataType.String), + ("duration", nix.DataType.Double)]) + + values = [("Alpha", "a", 0), + ("Beta", 'b', 0), + ("Gamma", 'c', 0), + ("Alpha", "a", 0), + ("Gamma", 'c', 0), + ("Alpha", "a", 0), + ("Gamma", 'c', 0), + ("Alpha", "a", 0), + ("Beta", 'b', 0)] + units = (None, None, "s") + df = self.block.create_data_frame("df-dimension", + "array.dimension.labels", + col_dict=column_descriptions, + data=values) + df.units = units + + self.range_dim.link_data_frame(df, 2) + assert self.range_dim.unit == df.units[2] + assert self.range_dim.label == df.column_names[2] + + self.range_dim.unit = "m" + assert df.units[2] == "m" + assert self.range_dim.unit == df.units[2] + + with self.assertRaises(RuntimeError): + # Can't change label: column name + self.range_dim.label = "a whole new label" From 86bdee5905f01ee4eddd737ba2f56f66445d81f6 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Sun, 19 Jul 2020 17:27:58 +0200 Subject: [PATCH 27/29] Code quality: Remove all unused imports and variables pylint warning codes W0611: Unused import W0612: Unused variable --- nixio/block.py | 2 +- nixio/data_array.py | 2 +- nixio/data_frame.py | 2 +- nixio/file.py | 4 ++-- nixio/section.py | 2 +- nixio/test/test_multi_tag.py | 6 +++--- nixio/test/xcompat/compile.py | 2 +- nixio/util/units.py | 8 ++++---- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/nixio/block.py b/nixio/block.py index 622f7b04..081f89ee 100644 --- a/nixio/block.py +++ b/nixio/block.py @@ -337,7 +337,7 @@ def create_data_frame(self, name="", type_="", col_dict=None, else: # if col_names is None if data is not None and type(data[0]) == np.void: col_dtype = data[0].dtype - for i, dt in enumerate(col_dtype.fields.values()): + for dt in col_dtype.fields.values(): cn = list(col_dtype.fields.keys()) raw_dt = col_dtype.fields.values() raw_dt = list(raw_dt) diff --git a/nixio/data_array.py b/nixio/data_array.py index 5aa56398..5b54752a 100644 --- a/nixio/data_array.py +++ b/nixio/data_array.py @@ -20,7 +20,7 @@ from . import util from .compression import Compression -from .exceptions import InvalidUnit, IncompatibleDimensions +from .exceptions import IncompatibleDimensions from .section import Section diff --git a/nixio/data_frame.py b/nixio/data_frame.py index a4ce238e..9bb9cb74 100644 --- a/nixio/data_frame.py +++ b/nixio/data_frame.py @@ -185,7 +185,7 @@ def write_rows(self, rows, index): self._write_data(rows, sl=index) else: cr_list = [] - for i, cr in enumerate(rows): + for cr in rows: cr_list.append(tuple(cr)) self._write_data(cr_list, sl=index) diff --git a/nixio/file.py b/nixio/file.py index a2985549..494e5a1b 100644 --- a/nixio/file.py +++ b/nixio/file.py @@ -46,8 +46,8 @@ def can_read(nixfile): filever = nixfile.version if len(filever) != 3: raise RuntimeError("Invalid version specified in file.") - vx, vy, vz = HDF_FF_VERSION - fx, fy, fz = filever + vx, vy, _ = HDF_FF_VERSION + fx, fy, _ = filever if vx == fx and vy >= fy: return True else: diff --git a/nixio/section.py b/nixio/section.py index d3a14ab5..ce5f1295 100644 --- a/nixio/section.py +++ b/nixio/section.py @@ -469,7 +469,7 @@ def __setitem__(self, key, data): prop.values = data def __iter__(self): - for name, item in self.items(): + for _, item in self.items(): yield item def items(self): diff --git a/nixio/test/test_multi_tag.py b/nixio/test/test_multi_tag.py index 628e8894..beb491f1 100644 --- a/nixio/test/test_multi_tag.py +++ b/nixio/test/test_multi_tag.py @@ -432,8 +432,8 @@ def test_multi_tag_tagged_data(self): assert (segdata.shape == (1, 5, 1)) # retrieve all positions for all references - for ridx, ref in enumerate(mtag.references): - for pidx, p in enumerate(mtag.positions): + for ridx, _ in enumerate(mtag.references): + for pidx, _ in enumerate(mtag.positions): mtag.tagged_data(pidx, ridx) wrong_pos = self.block.create_data_array("incorpos", "test", @@ -462,7 +462,7 @@ def test_multi_tag_tagged_data_1d(self): onedmtag = self.block.create_multi_tag("2dmt", "mtag", positions=onedpos) onedmtag.references.append(oneddata) - for pidx, p in enumerate(onedmtag.positions): + for pidx, _ in enumerate(onedmtag.positions): onedmtag.tagged_data(pidx, 0) def test_multi_tag_feature_data(self): diff --git a/nixio/test/xcompat/compile.py b/nixio/test/xcompat/compile.py index 39dbd3f2..cce15099 100644 --- a/nixio/test/xcompat/compile.py +++ b/nixio/test/xcompat/compile.py @@ -41,7 +41,7 @@ def cc(filenames, dest, objnames = compiler.compile(filenames, output_dir=dest, extra_postargs=compile_args) for obj in objnames: - execname, ext = os.path.splitext(obj) + execname, _ = os.path.splitext(obj) compiler.link_executable( [obj], execname, output_dir=dest, target_lang="c++", diff --git a/nixio/util/units.py b/nixio/util/units.py index 536f9cdb..8a59d59b 100644 --- a/nixio/util/units.py +++ b/nixio/util/units.py @@ -137,8 +137,8 @@ def scalable(unit_a, unit_b): if not (is_si(unit_a) and is_si(unit_b)): return False - a_prefix, a_unit, a_power = split(unit_a) - b_prefix, b_unit, b_power = split(unit_b) + _, a_unit, a_power = split(unit_a) + _, b_unit, b_power = split(unit_b) if a_unit != b_unit or a_power != b_power: return False @@ -163,8 +163,8 @@ def scaling(origin, destination): "nixio.util.scaling" ) - org_prefix, org_unit, org_power = split(origin) - dest_prefix, dest_unit, dest_power = split(destination) + org_prefix, _, org_power = split(origin) + dest_prefix, _, dest_power = split(destination) if org_prefix == dest_prefix and org_power == dest_power: return scale From 04063e79d5bf3b4b1522cc8df11962b548ed69bc Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Sun, 19 Jul 2020 17:44:50 +0200 Subject: [PATCH 28/29] Code quality: Remove unnecessary for loop --- nixio/block.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nixio/block.py b/nixio/block.py index 081f89ee..4c8bdb1b 100644 --- a/nixio/block.py +++ b/nixio/block.py @@ -337,11 +337,10 @@ def create_data_frame(self, name="", type_="", col_dict=None, else: # if col_names is None if data is not None and type(data[0]) == np.void: col_dtype = data[0].dtype - for dt in col_dtype.fields.values(): - cn = list(col_dtype.fields.keys()) - raw_dt = col_dtype.fields.values() - raw_dt = list(raw_dt) - raw_dt_list = [ele[0] for ele in raw_dt] + cn = list(col_dtype.fields.keys()) + raw_dt = col_dtype.fields.values() + raw_dt = list(raw_dt) + raw_dt_list = [ele[0] for ele in raw_dt] col_dict = OrderedDict(zip(cn, raw_dt_list)) if len(col_dtype.fields.values()) != len(col_dict): raise exceptions.DuplicateColumnName From 0ab71e7059bda002f93d611b96b71b88fd2d7708 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Mon, 20 Jul 2020 18:03:43 +0200 Subject: [PATCH 29/29] [travis] Extra build without compat tests --- .travis.yml | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index ce75e5d2..8fe27758 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,10 @@ matrix: env: pymajor=3 coverage=1 dist: xenial sudo: true + - python: "3.7" + os: linux + env: pymajor=3 nocompat=1 + dist: xenial - language: generic os: osx env: pymajor=2 @@ -56,25 +60,28 @@ addons: - libboost-all-dev before_install: - - nixprefix="/usr/local" - - export NIX_INCDIR=${nixprefix}/include/nixio-1.0 - - export NIX_LIBDIR=${nixprefix}/lib - - export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:${nixprefix}/lib/pkgconfig + # build nix for compat tests + - if [[ -z "${nocompat}" ]]; then + nixprefix="/usr/local"; + export NIX_INCDIR=${nixprefix}/include/nixio-1.0; + export NIX_LIBDIR=${nixprefix}/lib; + export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:${nixprefix}/lib/pkgconfig; + git clone --branch ${NIX_BRANCH} https://github.com/G-Node/nix /tmp/libnix; + pushd /tmp/libnix; + mkdir build; + pushd build; + cmake -DCMAKE_INSTALL_PREFIX=${nixprefix} ..; + make; + sudo make install; + popd; + popd; + fi # For macOS python3 - export PATH="/usr/local/opt/python@3.8/bin:$PATH" - alias pip2='pip' - if [[ "${TRAVIS_OS_NAME}" != "osx" ]]; then pip${pymajor} install --upgrade numpy; fi - pip${pymajor} install --upgrade h5py pytest pytest-xdist pytest-cov six; - if [[ "${pymajor}" == 2 ]]; then pip${pymajor} install enum34; fi - - git clone --branch ${NIX_BRANCH} https://github.com/G-Node/nix /tmp/libnix - - pushd /tmp/libnix - - mkdir build - - pushd build - - cmake -DCMAKE_INSTALL_PREFIX=${nixprefix} .. - - make - - sudo make install - - popd - - popd - which pip${pymajor} - which python${pymajor} - python${pymajor} --version @@ -83,7 +90,11 @@ install: - python${pymajor} setup.py build script: - - pytest --cov=nixio --nix-compat -nauto; + - if [[ "${nocompat}" == 1 ]]; then + pytest --cov=nixio -nauto; + else + pytest --cov=nixio --nix-compat -nauto; + fi after_success: - if [[ "${coverage}" == 1 ]]; then