Skip to content

Commit

Permalink
Support no_data_value / fill_value in data model
Browse files Browse the repository at this point in the history
  • Loading branch information
@kordejong
kordejong committed Mar 14, 2024
1 parent 796d68e commit 4a24377
Show file tree
Hide file tree
Showing 17 changed files with 1,374 additions and 1,197 deletions.
4 changes: 2 additions & 2 deletions source/data_model/cxx/include/lue/array/different_shape/value.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,15 @@ namespace lue {

void expand(Count nr_objects, ID const* ids, hdf5::Shape::value_type const* shapes);

void expand(ID id, hdf5::Shape const& shape);
void expand(ID id, hdf5::Shape const& shape, void const* no_data_value = nullptr);

bool contains(ID id) const;

Array operator[](ID id) const;

private:

void expand_core(ID id, hdf5::Shape const& shape);
void expand_core(ID id, hdf5::Shape const& shape, void const* no_data_value = nullptr);

Count _nr_objects;
};
Expand Down
14 changes: 10 additions & 4 deletions source/data_model/cxx/include/lue/array/same_shape/constant_shape/value.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,26 +71,32 @@ namespace lue {


Value create_value(
hdf5::Group& parent, std::string const& name, hdf5::Datatype const& memory_datatype);
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& memory_datatype,
void const* no_data_value = nullptr);

Value create_value(
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& memory_datatype,
hdf5::Shape const& array_shape);
hdf5::Shape const& array_shape,
void const* no_data_value = nullptr);

Value create_value(
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& file_datatype,
hdf5::Datatype const& memory_datatype);
hdf5::Datatype const& memory_datatype,
void const* no_data_value = nullptr);

Value create_value(
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& file_datatype,
hdf5::Datatype const& memory_datatype,
hdf5::Shape const& array_shape);
hdf5::Shape const& array_shape,
void const* no_data_value = nullptr);

bool value_exists(hdf5::Group const& parent, std::string const& name);

Expand Down
14 changes: 10 additions & 4 deletions source/data_model/cxx/include/lue/array/same_shape/value.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,26 +68,32 @@ namespace lue {


Value create_value(
hdf5::Group& parent, std::string const& name, hdf5::Datatype const& memory_datatype);
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& memory_datatype,
void const* no_data_value = nullptr);

Value create_value(
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& memory_datatype,
hdf5::Shape const& array_shape);
hdf5::Shape const& array_shape,
void const* no_data_value = nullptr);

Value create_value(
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& file_datatype,
hdf5::Datatype const& memory_datatype);
hdf5::Datatype const& memory_datatype,
void const* no_data_value = nullptr);

Value create_value(
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& file_datatype,
hdf5::Datatype const& memory_datatype,
hdf5::Shape const& array_shape);
hdf5::Shape const& array_shape,
void const* no_data_value = nullptr);

} // namespace same_shape
} // namespace data_model
Expand Down
17 changes: 17 additions & 0 deletions source/data_model/cxx/include/lue/core/array.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,23 @@ namespace lue {
TransferPropertyList const& transfer_property_list,
void const* buffer);


template<typename Element>
void set_no_data_value(Element const no_data_value)
{
creation_property_list().set_fill_value<Element>(no_data_value);
}


template<typename Element>
auto no_data_value() const -> Element
{
return creation_property_list().get_fill_value<Element>();
}


auto has_no_data_value() const -> bool;

protected:

private:
Expand Down
11 changes: 8 additions & 3 deletions source/data_model/cxx/src/array/different_shape/value.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,9 @@ namespace lue {
@param id The object ID
@param shape The shape of the object array
*/
void Value::expand(ID const id, hdf5::Shape const& shape)
void Value::expand(ID const id, hdf5::Shape const& shape, void const* no_data_value)
{
expand_core(id, shape);
expand_core(id, shape, no_data_value);

_nr_objects += 1;

Expand All @@ -111,7 +111,7 @@ namespace lue {
@param id The object ID
@param shape The shape of the object array
*/
void Value::expand_core(ID const id, hdf5::Shape const& array_shape)
void Value::expand_core(ID const id, hdf5::Shape const& array_shape, void const* no_data_value)
{
assert(!contains(id));

Expand All @@ -125,6 +125,11 @@ namespace lue {
// hdf5::chunk_shape(array_shape, file_datatype.size());
// creation_property_list.set_chunk(chunk_dimension_sizes);

if (no_data_value)
{
creation_property_list.set_fill_value(memory_datatype(), no_data_value);
}

/* auto dataset = */ hdf5::create_dataset(
this->id(), name, file_datatype(), dataspace, creation_property_list);
}
Expand Down
36 changes: 29 additions & 7 deletions source/data_model/cxx/src/array/same_shape/constant_shape/value.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,10 +137,18 @@ namespace lue {
@brief Create value @a name in @a parent
*/
Value create_value(
hdf5::Group& parent, std::string const& name, hdf5::Datatype const& memory_datatype)
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& memory_datatype,
void const* no_data_value)
{
return create_value(
parent, name, file_datatype(memory_datatype), memory_datatype, hdf5::Shape{});
parent,
name,
file_datatype(memory_datatype),
memory_datatype,
hdf5::Shape{},
no_data_value);
}


Expand All @@ -151,10 +159,16 @@ namespace lue {
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& memory_datatype,
hdf5::Shape const& array_shape)
hdf5::Shape const& array_shape,
void const* no_data_value)
{
return create_value(
parent, name, file_datatype(memory_datatype), memory_datatype, array_shape);
parent,
name,
file_datatype(memory_datatype),
memory_datatype,
array_shape,
no_data_value);
}


Expand All @@ -165,9 +179,11 @@ namespace lue {
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& file_datatype,
hdf5::Datatype const& memory_datatype)
hdf5::Datatype const& memory_datatype,
void const* no_data_value)
{
return create_value(parent, name, file_datatype, memory_datatype, hdf5::Shape{});
return create_value(
parent, name, file_datatype, memory_datatype, hdf5::Shape{}, no_data_value);
}


Expand All @@ -184,7 +200,8 @@ namespace lue {
std::string const& name,
hdf5::Datatype const& file_datatype,
hdf5::Datatype const& memory_datatype,
hdf5::Shape const& array_shape)
hdf5::Shape const& array_shape,
void const* no_data_value)
{
// The rank of the underlying dataset is one larger than the rank of the
// object arrays. Object arrays are stored one after the other.
Expand All @@ -202,6 +219,11 @@ namespace lue {
hdf5::chunk_shape(array_shape, file_datatype.size())};
creation_property_list.set_chunk(chunk_dimension_sizes);

if (no_data_value)
{
creation_property_list.set_fill_value(memory_datatype, no_data_value);
}

hdf5::Dataset dataset{hdf5::create_dataset(
parent.id(), name, file_datatype, dataspace, creation_property_list)};

Expand Down
36 changes: 29 additions & 7 deletions source/data_model/cxx/src/array/same_shape/value.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,10 +128,18 @@ namespace lue {
@brief Create value @a name in @a parent
*/
Value create_value(
hdf5::Group& parent, std::string const& name, hdf5::Datatype const& memory_datatype)
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& memory_datatype,
void const* no_data_value)
{
return create_value(
parent, name, file_datatype(memory_datatype), memory_datatype, hdf5::Shape{});
parent,
name,
file_datatype(memory_datatype),
memory_datatype,
hdf5::Shape{},
no_data_value);
}


Expand All @@ -142,10 +150,16 @@ namespace lue {
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& memory_datatype,
hdf5::Shape const& array_shape)
hdf5::Shape const& array_shape,
void const* no_data_value)
{
return create_value(
parent, name, file_datatype(memory_datatype), memory_datatype, array_shape);
parent,
name,
file_datatype(memory_datatype),
memory_datatype,
array_shape,
no_data_value);
}


Expand All @@ -156,9 +170,11 @@ namespace lue {
hdf5::Group& parent,
std::string const& name,
hdf5::Datatype const& file_datatype,
hdf5::Datatype const& memory_datatype)
hdf5::Datatype const& memory_datatype,
void const* no_data_value)
{
return create_value(parent, name, file_datatype, memory_datatype, hdf5::Shape{});
return create_value(
parent, name, file_datatype, memory_datatype, hdf5::Shape{}, no_data_value);
}


Expand All @@ -175,7 +191,8 @@ namespace lue {
std::string const& name,
hdf5::Datatype const& file_datatype,
hdf5::Datatype const& memory_datatype,
hdf5::Shape const& array_shape)
hdf5::Shape const& array_shape,
void const* no_data_value)
{
// The rank of the underlying dataset is one larger than the rank of the
// object arrays. Object arrays are stored one after the other.
Expand All @@ -191,6 +208,11 @@ namespace lue {
auto chunk_dimension_sizes = hdf5::chunk_shape(array_shape, file_datatype.size());
creation_property_list.set_chunk(chunk_dimension_sizes);

if (no_data_value)
{
creation_property_list.set_fill_value(memory_datatype, no_data_value);
}

hdf5::Dataset dataset{hdf5::create_dataset(
parent.id(), name, file_datatype, dataspace, creation_property_list)};

Expand Down
6 changes: 6 additions & 0 deletions source/data_model/cxx/src/core/array.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,5 +137,11 @@ namespace lue {
_memory_datatype, memory_dataspace, hyperslab, transfer_property_list, buffer);
}


auto Array::has_no_data_value() const -> bool
{
return creation_property_list().fill_value_defined();
}

} // namespace data_model
} // namespace lue
50 changes: 50 additions & 0 deletions source/data_model/hdf5/include/lue/hdf5/dataset.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,57 @@ namespace lue {

CreationPropertyList();

CreationPropertyList(Identifier&& property_list_id);

void set_chunk(Shape const& chunk);

void set_fill_time(H5D_fill_time_t fill_time);

void set_fill_value(Datatype const& datatype, void const* value);

template<typename Element>
void set_fill_value(Element const value)
{
auto const datatype = []()
{
if constexpr (std::is_same_v<Element, bool>)
{
return hdf5::native_datatype<std::uint8_t>();
}
else
{
return hdf5::native_datatype<Element>();
}
}();

set_fill_value(datatype, &value);
}

void get_fill_value(Datatype const& datatype, void* value) const;

template<typename Element>
auto get_fill_value() const -> Element
{
auto const datatype = []()
{
if constexpr (std::is_same_v<Element, bool>)
{
return hdf5::native_datatype<std::uint8_t>();
}
else
{
return hdf5::native_datatype<Element>();
}
}();

Element fill_value;

get_fill_value(datatype, &fill_value);

return fill_value;
}

auto fill_value_defined() const -> bool;
};


Expand Down Expand Up @@ -131,6 +179,8 @@ namespace lue {

void fill(Datatype const& datatype, Hyperslab const& hyperslab, void const* buffer) const;

auto creation_property_list() const -> CreationPropertyList;

private:
};

Expand Down
Loading

0 comments on commit 4a24377

Please sign in to comment.