From b3f34bd95d0bdd357dfcb8a34174ef5f1301af05 Mon Sep 17 00:00:00 2001 From: Luc Grosheintz Date: Tue, 14 May 2024 12:28:49 +0200 Subject: [PATCH] No constexpr rank via constexpr min-max rank. (#938) The commit removes the requirement for a constexpr rank. Instead containers need a minimum and maximum constexpr rank. The difficulties are: * Empty arrays can't figure out their runtime rank. This is an issue when deducing the dimension when writing to disk. The "solution" is to deduce with lowest rank possible. --- include/highfive/bits/H5Attribute_misc.hpp | 15 ++-- include/highfive/bits/H5DataType_misc.hpp | 6 +- include/highfive/bits/H5Dataspace_misc.hpp | 7 +- include/highfive/bits/H5Inspector_misc.hpp | 84 ++++++++++++++----- include/highfive/bits/H5ReadWrite_misc.hpp | 27 +++++- include/highfive/bits/H5Slice_traits_misc.hpp | 13 +-- include/highfive/boost.hpp | 30 +++++-- include/highfive/eigen.hpp | 9 +- include/highfive/span.hpp | 17 +++- tests/unit/data_generator.hpp | 9 +- 10 files changed, 163 insertions(+), 54 deletions(-) diff --git a/include/highfive/bits/H5Attribute_misc.hpp b/include/highfive/bits/H5Attribute_misc.hpp index d339af1f1..7ad38699a 100644 --- a/include/highfive/bits/H5Attribute_misc.hpp +++ b/include/highfive/bits/H5Attribute_misc.hpp @@ -19,6 +19,7 @@ #include "../H5DataSpace.hpp" #include "H5Converter_misc.hpp" +#include "H5Inspector_misc.hpp" #include "H5ReadWrite_misc.hpp" #include "H5Utils.hpp" #include "h5a_wrapper.hpp" @@ -73,10 +74,11 @@ inline void Attribute::read(T& array) const { [this]() -> std::string { return this->getName(); }, details::BufferInfo::Operation::read); - if (!details::checkDimensions(mem_space, buffer_info.n_dimensions)) { + if (!details::checkDimensions(mem_space, buffer_info.getMinRank(), buffer_info.getMaxRank())) { std::ostringstream ss; - ss << "Impossible to read Attribute of dimensions " << mem_space.getNumberDimensions() - << " into arrays of dimensions " << buffer_info.n_dimensions; + ss << "Impossible to read attribute of dimensions " << mem_space.getNumberDimensions() + << " into arrays of dimensions: " << buffer_info.getMinRank() << "(min) to " + << buffer_info.getMaxRank() << "(max)"; throw DataSpaceException(ss.str()); } auto dims = mem_space.getDimensions(); @@ -137,10 +139,11 @@ inline void Attribute::write(const T& buffer) { [this]() -> std::string { return this->getName(); }, details::BufferInfo::Operation::write); - if (!details::checkDimensions(mem_space, buffer_info.n_dimensions)) { + if (!details::checkDimensions(mem_space, buffer_info.getMinRank(), buffer_info.getMaxRank())) { std::ostringstream ss; - ss << "Impossible to write buffer of dimensions " << buffer_info.n_dimensions - << " into dataset of dimensions " << mem_space.getNumberDimensions(); + ss << "Impossible to write attribute of dimensions " << mem_space.getNumberDimensions() + << " into arrays of dimensions: " << buffer_info.getMinRank() << "(min) to " + << buffer_info.getMaxRank() << "(max)"; throw DataSpaceException(ss.str()); } auto w = details::data_converter::serialize(buffer, dims, file_datatype); diff --git a/include/highfive/bits/H5DataType_misc.hpp b/include/highfive/bits/H5DataType_misc.hpp index 4321a4658..797702188 100644 --- a/include/highfive/bits/H5DataType_misc.hpp +++ b/include/highfive/bits/H5DataType_misc.hpp @@ -226,9 +226,9 @@ inline EnumType create_enum_boolean() { // Other cases not supported. Fail early with a user message template AtomicType::AtomicType() { - static_assert(details::inspector::recursive_ndim == 0, - "Atomic types cant be arrays, except for char[] (fixed-length strings)"); - static_assert(details::inspector::recursive_ndim > 0, "Type not supported"); + static_assert( + true, + "Missing specialization of AtomicType. Therefore, type T is not supported by HighFive."); } diff --git a/include/highfive/bits/H5Dataspace_misc.hpp b/include/highfive/bits/H5Dataspace_misc.hpp index ceae1e531..4382c14c1 100644 --- a/include/highfive/bits/H5Dataspace_misc.hpp +++ b/include/highfive/bits/H5Dataspace_misc.hpp @@ -131,9 +131,10 @@ inline DataSpace DataSpace::FromCharArrayStrings(const char (&)[N][Width]) { namespace details { -/// dimension checks @internal -inline bool checkDimensions(const DataSpace& mem_space, size_t n_dim_requested) { - return checkDimensions(mem_space.getDimensions(), n_dim_requested); +inline bool checkDimensions(const DataSpace& mem_space, + size_t min_dim_requested, + size_t max_dim_requested) { + return checkDimensions(mem_space.getDimensions(), min_dim_requested, max_dim_requested); } } // namespace details diff --git a/include/highfive/bits/H5Inspector_misc.hpp b/include/highfive/bits/H5Inspector_misc.hpp index efb2343da..59bf85422 100644 --- a/include/highfive/bits/H5Inspector_misc.hpp +++ b/include/highfive/bits/H5Inspector_misc.hpp @@ -27,15 +27,18 @@ namespace HighFive { namespace details { -inline bool checkDimensions(const std::vector& dims, size_t n_dim_requested) { - if (dims.size() == n_dim_requested) { +inline bool checkDimensions(const std::vector& dims, + size_t min_dim_requested, + size_t max_dim_requested) { + if (min_dim_requested <= dims.size() && dims.size() <= max_dim_requested) { return true; } + // Scalar values still support broadcasting // into arrays with one element. size_t n_elements = compute_total_size(dims); - return n_elements == 1 && n_dim_requested == 0; + return n_elements == 1 && min_dim_requested == 0; } } // namespace details @@ -49,8 +52,6 @@ inspector { // hdf5_type is the base read by hdf5 (c-type) (e.g. std::vector => const char*) using hdf5_type - // Number of dimensions starting from here - static constexpr size_t recursive_ndim // Is the inner type trivially copyable for optimisation // If this value is true: data() is mandatory // If this value is false: serialize, unserialize are mandatory @@ -88,10 +89,16 @@ struct type_helper { using hdf5_type = base_type; static constexpr size_t ndim = 0; - static constexpr size_t recursive_ndim = ndim; + static constexpr size_t min_ndim = ndim; + static constexpr size_t max_ndim = ndim; + static constexpr bool is_trivially_copyable = std::is_trivially_copyable::value; static constexpr bool is_trivially_nestable = is_trivially_copyable; + static size_t getRank(const type& /* val */) { + return ndim; + } + static std::vector getDimensions(const type& /* val */) { return {}; } @@ -216,17 +223,27 @@ struct inspector> { using hdf5_type = typename inspector::hdf5_type; static constexpr size_t ndim = 1; - static constexpr size_t recursive_ndim = ndim + inspector::recursive_ndim; + static constexpr size_t min_ndim = ndim + inspector::min_ndim; + static constexpr size_t max_ndim = ndim + inspector::max_ndim; + static constexpr bool is_trivially_copyable = std::is_trivially_copyable::value && inspector::is_trivially_nestable; static constexpr bool is_trivially_nestable = false; + static size_t getRank(const type& val) { + if (!val.empty()) { + return ndim + inspector::getRank(val[0]); + } else { + return min_ndim; + } + } + static std::vector getDimensions(const type& val) { - std::vector sizes(recursive_ndim, 1ul); + auto rank = getRank(val); + std::vector sizes(rank, 1ul); sizes[0] = val.size(); if (!val.empty()) { auto s = inspector::getDimensions(val[0]); - assert(s.size() + ndim == sizes.size()); for (size_t i = 0; i < s.size(); ++i) { sizes[i + ndim] = s[i]; } @@ -280,10 +297,16 @@ struct inspector> { using hdf5_type = uint8_t; static constexpr size_t ndim = 1; - static constexpr size_t recursive_ndim = ndim; + static constexpr size_t min_ndim = ndim; + static constexpr size_t max_ndim = ndim; + static constexpr bool is_trivially_copyable = false; static constexpr bool is_trivially_nestable = false; + static size_t getRank(const type& /* val */) { + return ndim; + } + static std::vector getDimensions(const type& val) { std::vector sizes{val.size()}; return sizes; @@ -327,18 +350,22 @@ struct inspector> { using hdf5_type = typename inspector::hdf5_type; static constexpr size_t ndim = 1; - static constexpr size_t recursive_ndim = ndim + inspector::recursive_ndim; + static constexpr size_t min_ndim = ndim + inspector::min_ndim; + static constexpr size_t max_ndim = ndim + inspector::max_ndim; + static constexpr bool is_trivially_copyable = std::is_trivially_copyable::value && inspector::is_trivially_nestable; static constexpr bool is_trivially_nestable = (sizeof(type) == N * sizeof(T)) && is_trivially_copyable; + static size_t getRank(const type& val) { + return ndim + inspector::getRank(val[0]); + } + static std::vector getDimensions(const type& val) { std::vector sizes{N}; - if (!val.empty()) { - auto s = inspector::getDimensions(val[0]); - sizes.insert(sizes.end(), s.begin(), s.end()); - } + auto s = inspector::getDimensions(val[0]); + sizes.insert(sizes.end(), s.begin(), s.end()); return sizes; } @@ -399,11 +426,21 @@ struct inspector { using hdf5_type = typename inspector::hdf5_type; static constexpr size_t ndim = 1; - static constexpr size_t recursive_ndim = ndim + inspector::recursive_ndim; + static constexpr size_t min_ndim = ndim + inspector::min_ndim; + static constexpr size_t max_ndim = ndim + inspector::max_ndim; + static constexpr bool is_trivially_copyable = std::is_trivially_copyable::value && inspector::is_trivially_nestable; static constexpr bool is_trivially_nestable = false; + static size_t getRank(const type& val) { + if (val != nullptr) { + return ndim + inspector::getRank(val[0]); + } else { + return min_ndim; + } + } + static std::vector getDimensions(const type& /* val */) { throw DataSpaceException("Not possible to have size of a T*"); } @@ -430,7 +467,9 @@ struct inspector { using hdf5_type = typename inspector::hdf5_type; static constexpr size_t ndim = 1; - static constexpr size_t recursive_ndim = ndim + inspector::recursive_ndim; + static constexpr size_t min_ndim = ndim + inspector::min_ndim; + static constexpr size_t max_ndim = ndim + inspector::max_ndim; + static constexpr bool is_trivially_copyable = std::is_trivially_copyable::value && inspector::is_trivially_nestable; static constexpr bool is_trivially_nestable = is_trivially_copyable; @@ -450,12 +489,14 @@ struct inspector { } } + static size_t getRank(const type& val) { + return ndim + inspector::getRank(val[0]); + } + static std::vector getDimensions(const type& val) { std::vector sizes{N}; - if (N > 0) { - auto s = inspector::getDimensions(val[0]); - sizes.insert(sizes.end(), s.begin(), s.end()); - } + auto s = inspector::getDimensions(val[0]); + sizes.insert(sizes.end(), s.begin(), s.end()); return sizes; } @@ -478,5 +519,6 @@ struct inspector { } }; + } // namespace details } // namespace HighFive diff --git a/include/highfive/bits/H5ReadWrite_misc.hpp b/include/highfive/bits/H5ReadWrite_misc.hpp index 05bb49888..e5c862bc5 100644 --- a/include/highfive/bits/H5ReadWrite_misc.hpp +++ b/include/highfive/bits/H5ReadWrite_misc.hpp @@ -9,6 +9,7 @@ #pragma once #include +#include "H5Inspector_misc.hpp" #include "H5Utils.hpp" namespace HighFive { @@ -57,10 +58,14 @@ struct BufferInfo { template BufferInfo(const DataType& dtype, F getName, Operation _op); + size_t getRank(const T& array) const; + size_t getMinRank() const; + size_t getMaxRank() const; + // member data for info depending on the destination dataset type const bool is_fixed_len_string; - const size_t n_dimensions; const DataType data_type; + const size_t rank_correction; }; // details implementation @@ -135,10 +140,9 @@ BufferInfo::BufferInfo(const DataType& file_data_type, F getName, Operation _ : op(_op) , is_fixed_len_string(file_data_type.isFixedLenStr()) // In case we are using Fixed-len strings we need to subtract one dimension - , n_dimensions(details::inspector::recursive_ndim - - ((is_fixed_len_string && is_char_array) ? 1 : 0)) , data_type(string_type_checker::getDataType(create_datatype(), - file_data_type)) { + file_data_type)) + , rank_correction((is_fixed_len_string && is_char_array) ? 1 : 0) { // We warn. In case they are really not convertible an exception will rise on read/write if (file_data_type.getClass() != data_type.getClass()) { HIGHFIVE_LOG_WARN(getName() + "\": data and hdf5 dataset have different types: " + @@ -157,6 +161,21 @@ BufferInfo::BufferInfo(const DataType& file_data_type, F getName, Operation _ } } +template +size_t BufferInfo::getRank(const T& array) const { + return details::inspector::getRank(array) - rank_correction; +} + +template +size_t BufferInfo::getMinRank() const { + return details::inspector::min_ndim - rank_correction; +} + +template +size_t BufferInfo::getMaxRank() const { + return details::inspector::max_ndim - rank_correction; +} + } // namespace details } // namespace HighFive diff --git a/include/highfive/bits/H5Slice_traits_misc.hpp b/include/highfive/bits/H5Slice_traits_misc.hpp index bf3f789eb..711d57ac2 100644 --- a/include/highfive/bits/H5Slice_traits_misc.hpp +++ b/include/highfive/bits/H5Slice_traits_misc.hpp @@ -180,10 +180,11 @@ inline void SliceTraits::read(T& array, const DataTransferProps& xfer_ [&slice]() -> std::string { return details::get_dataset(slice).getPath(); }, details::BufferInfo::Operation::read); - if (!details::checkDimensions(mem_space, buffer_info.n_dimensions)) { + if (!details::checkDimensions(mem_space, buffer_info.getMinRank(), buffer_info.getMaxRank())) { std::ostringstream ss; ss << "Impossible to read DataSet of dimensions " << mem_space.getNumberDimensions() - << " into arrays of dimensions " << buffer_info.n_dimensions; + << " into arrays of dimensions: " << buffer_info.getMinRank() << "(min) to " + << buffer_info.getMaxRank() << "(max)"; throw DataSpaceException(ss.str()); } auto dims = mem_space.getDimensions(); @@ -254,11 +255,11 @@ inline void SliceTraits::write(const T& buffer, const DataTransferProp [&slice]() -> std::string { return details::get_dataset(slice).getPath(); }, details::BufferInfo::Operation::write); - if (!details::checkDimensions(mem_space, buffer_info.n_dimensions)) { + if (!details::checkDimensions(mem_space, buffer_info.getMinRank(), buffer_info.getMaxRank())) { std::ostringstream ss; - ss << "Impossible to write buffer of dimensions " - << details::format_vector(mem_space.getDimensions()) - << " into dataset with n = " << buffer_info.n_dimensions << " dimensions."; + ss << "Impossible to write buffer with dimensions n = " << buffer_info.getRank(buffer) + << "into dataset with dimensions " << details::format_vector(mem_space.getDimensions()) + << "."; throw DataSpaceException(ss.str()); } auto w = details::data_converter::serialize(buffer, dims, file_datatype); diff --git a/include/highfive/boost.hpp b/include/highfive/boost.hpp index 3e42a5b60..33c1458df 100644 --- a/include/highfive/boost.hpp +++ b/include/highfive/boost.hpp @@ -17,19 +17,31 @@ struct inspector> { using hdf5_type = typename inspector::hdf5_type; static constexpr size_t ndim = Dims; - static constexpr size_t recursive_ndim = ndim + inspector::recursive_ndim; + static constexpr size_t min_ndim = ndim + inspector::min_ndim; + static constexpr size_t max_ndim = ndim + inspector::max_ndim; + static constexpr bool is_trivially_copyable = std::is_trivially_copyable::value && inspector::is_trivially_nestable; static constexpr bool is_trivially_nestable = false; + static size_t getRank(const type& val) { + return ndim + inspector::getRank(val.data()[0]); + } + static std::vector getDimensions(const type& val) { - std::vector sizes; + auto rank = getRank(val); + std::vector sizes(rank, 1ul); for (size_t i = 0; i < ndim; ++i) { - sizes.push_back(val.shape()[i]); + sizes[i] = val.shape()[i]; + } + if (val.size() != 0) { + auto s = inspector::getDimensions(val.data()[0]); + sizes.resize(ndim + s.size()); + for (size_t i = 0; i < s.size(); ++i) { + sizes[ndim + i] = s[i]; + } } - auto s = inspector::getDimensions(val.data()[0]); - sizes.insert(sizes.end(), s.begin(), s.end()); return sizes; } @@ -101,11 +113,17 @@ struct inspector> { using hdf5_type = typename inspector::hdf5_type; static constexpr size_t ndim = 2; - static constexpr size_t recursive_ndim = ndim + inspector::recursive_ndim; + static constexpr size_t min_ndim = ndim + inspector::min_ndim; + static constexpr size_t max_ndim = ndim + inspector::max_ndim; + static constexpr bool is_trivially_copyable = std::is_trivially_copyable::value && inspector::is_trivially_copyable; static constexpr bool is_trivially_nestable = false; + static size_t getRank(const type& val) { + return ndim + inspector::getRank(val(0, 0)); + } + static std::vector getDimensions(const type& val) { std::vector sizes{val.size1(), val.size2()}; auto s = inspector::getDimensions(val(0, 0)); diff --git a/include/highfive/eigen.hpp b/include/highfive/eigen.hpp index 4a0b293fd..462769e4b 100644 --- a/include/highfive/eigen.hpp +++ b/include/highfive/eigen.hpp @@ -16,6 +16,7 @@ struct eigen_inspector { using base_type = typename inspector::base_type; using hdf5_type = base_type; + static_assert(int(EigenType::ColsAtCompileTime) == int(EigenType::MaxColsAtCompileTime), "Padding isn't supported."); static_assert(int(EigenType::RowsAtCompileTime) == int(EigenType::MaxRowsAtCompileTime), @@ -26,13 +27,19 @@ struct eigen_inspector { EigenType::IsRowMajor; } + static constexpr size_t ndim = 2; - static constexpr size_t recursive_ndim = ndim + inspector::recursive_ndim; + static constexpr size_t min_ndim = ndim + inspector::min_ndim; + static constexpr size_t max_ndim = ndim + inspector::max_ndim; static constexpr bool is_trivially_copyable = is_row_major() && std::is_trivially_copyable::value && inspector::is_trivially_nestable; static constexpr bool is_trivially_nestable = false; + static size_t getRank(const type& val) { + return ndim + inspector::getRank(val.data()[0]); + } + static std::vector getDimensions(const type& val) { std::vector sizes{static_cast(val.rows()), static_cast(val.cols())}; auto s = inspector::getDimensions(val.data()[0]); diff --git a/include/highfive/span.hpp b/include/highfive/span.hpp index 1eca4a51b..ab53319ee 100644 --- a/include/highfive/span.hpp +++ b/include/highfive/span.hpp @@ -25,14 +25,25 @@ struct inspector> { using hdf5_type = typename inspector::hdf5_type; static constexpr size_t ndim = 1; - static constexpr size_t recursive_ndim = ndim + inspector::recursive_ndim; + static constexpr size_t min_ndim = ndim + inspector::min_ndim; + static constexpr size_t max_ndim = ndim + inspector::max_ndim; + static constexpr bool is_trivially_copyable = std::is_trivially_copyable::value && inspector::is_trivially_nestable; - static constexpr bool is_trivially_nestable = false; + + static size_t getRank(const type& val) { + if (!val.empty()) { + return ndim + inspector::getRank(val[0]); + } else { + return min_ndim; + } + } + static std::vector getDimensions(const type& val) { - std::vector sizes(recursive_ndim, 1ul); + auto rank = getRank(val); + std::vector sizes(rank, 1ul); sizes[0] = val.size(); if (!val.empty()) { auto s = inspector::getDimensions(val[0]); diff --git a/tests/unit/data_generator.hpp b/tests/unit/data_generator.hpp index 5b1f095ca..a50284d5c 100644 --- a/tests/unit/data_generator.hpp +++ b/tests/unit/data_generator.hpp @@ -79,6 +79,7 @@ struct ScalarContainerTraits { using base_type = T; static constexpr bool is_view = false; + static constexpr size_t rank = 0; static void set(container_type& array, std::vector /* indices */, base_type value) { array = value; @@ -120,6 +121,7 @@ struct ContainerTraits> { using base_type = bool; static constexpr bool is_view = false; + static constexpr size_t rank = 1; static void set(container_type& array, const std::vector& indices, @@ -154,6 +156,7 @@ struct STLLikeContainerTraits { using base_type = typename ContainerTraits::base_type; static constexpr bool is_view = ContainerTraits::is_view; + static constexpr size_t rank = 1 + ContainerTraits::rank; static void set(container_type& array, const std::vector& indices, @@ -281,6 +284,7 @@ struct ContainerTraits> { using base_type = typename ContainerTraits::base_type; static constexpr bool is_view = ContainerTraits::is_view; + static constexpr size_t rank = n + ContainerTraits::rank; static void set(container_type& array, const std::vector& indices, @@ -333,6 +337,7 @@ struct ContainerTraits> { using base_type = typename ContainerTraits::base_type; static constexpr bool is_view = ContainerTraits::is_view; + static constexpr size_t rank = 2 + ContainerTraits::rank; static void set(container_type& array, const std::vector& indices, @@ -392,6 +397,7 @@ struct EigenContainerTraits { using base_type = typename ContainerTraits::base_type; static constexpr bool is_view = ContainerTraits::is_view; + static constexpr size_t rank = 2 + ContainerTraits::rank; static void set(container_type& array, const std::vector& indices, @@ -629,11 +635,12 @@ struct MultiDimVector { template class DataGenerator { public: - constexpr static size_t rank = details::inspector::recursive_ndim; using traits = ContainerTraits; using base_type = typename traits::base_type; using container_type = Container; + constexpr static size_t rank = traits::rank; + public: static container_type allocate(const std::vector& dims) { return traits::allocate(dims);