From 76382329c47c94924eb97a2100a0116494456bee Mon Sep 17 00:00:00 2001
From: Luc Grosheintz <luc.grosheintz@gmail.com>
Date: Tue, 19 Dec 2023 15:04:39 +0100
Subject: [PATCH] Improve testing infrastructure. (#871)

Adds/reimplements abstractions for the following:

  * Create multi-dimensional array filled with suitable values.
  * Traits for accessing values.
  * Traits for hiding the difference of DataSet and Attribute.
  * Useful utilities such as `ravel`, `unravel` and `flat_size`.
---
 doc/developer_guide.md                    | 131 ++++++
 tests/unit/create_traits.hpp              |  70 ++++
 tests/unit/data_generator.hpp             | 463 ++++++++++++++++++++++
 tests/unit/supported_types.hpp            | 108 +++++
 tests/unit/test_all_types.cpp             | 246 ++++++++++++
 tests/unit/tests_high_five_multi_dims.cpp |   2 +-
 6 files changed, 1019 insertions(+), 1 deletion(-)
 create mode 100644 tests/unit/create_traits.hpp
 create mode 100644 tests/unit/data_generator.hpp
 create mode 100644 tests/unit/supported_types.hpp

diff --git a/doc/developer_guide.md b/doc/developer_guide.md
index 3017289b5..fc388f3b5 100644
--- a/doc/developer_guide.md
+++ b/doc/developer_guide.md
@@ -91,3 +91,134 @@ release. Once this is done perform a final round of updates:
 * Update BlueBrain Spack recipe to use the archive and not the Git commit.
 * Update the upstream Spack recipe.
 
+## Writing Tests
+### Generate Multi-Dimensional Test Data
+Input array of any dimension and type can be generated using the template class
+`DataGenerator`. For example:
+```
+auto dims = std::vector<size_t>{4, 2};
+auto values = testing::DataGenerator<std::vector<std::array<double, 2>>::create(dims);
+```
+Generates an `std::vector<std::array<double, 2>>` initialized with suitable
+values.
+
+If "suitable" isn't specific enough, one can specify a callback:
+```
+auto callback = [](const std::vector<size_t>& indices) {
+    return 42.0;
+}
+
+auto values = testing::DataGenerator<std::vector<double>>::create(dims, callback);
+```
+
+The `dims` can be generated via `testing::DataGenerator::default_dims` or by
+using `testing::DataGenerator::sanitize_dims`. Remember, that certain
+containers are fixed size and that we often compute the number of elements by
+multiplying the dims.
+
+### Generate Scalar Test Data
+To generate a single "suitable" element use template class `DefaultValues`, e.g.
+```
+auto default_values = testing::DefaultValues<double>();
+auto x = testing::DefaultValues<double>(indices);
+```
+
+### Accessing Elements
+To access a particular element from an unknown container use the following trait:
+```
+using trait = testing::ContainerTraits<std::vector<std::array<int, 2>>;
+// auto x = values[1][0];
+auto x = trait::get(values, {1, 0});
+
+// values[1][0] = 42.0;
+trait::set(values, {1, 0}, 42.0);
+```
+
+### Utilities For Multi-Dimensional Arrays
+Use `testing::DataGenerator::allocate` to allocate an array (without filling
+it) and `testing::copy` to copy an array from one type to another. There's
+`testing::ravel`, `testing::unravel` and `testing::flat_size` to compute the
+position in a flat array from a multi-dimensional index, the reverse and the
+number of element in the multi-dimensional array.
+
+### Deduplicating DataSet and Attribute
+Due to how HighFive is written testing `DataSet` and `Attribute` often requires
+duplicating the entire test code because somewhere a `createDataSet` must be
+replaced with `createAttribute`. Use `testing::AttributeCreateTraits` and
+`testing::DataSetCreateTraits`. For example,
+```
+template<class CreateTraits>
+void check_write(...) {
+    // Same as one of:
+    //   file.createDataSet(name, values);
+    //   file.createAttribute(name, values);
+    CreateTraits::create(file, name, values);
+}
+```
+
+### Test Organization
+#### Multi-Dimensional Arrays
+All tests for reading/writing whole multi-dimensional arrays to datasets or
+attributes belong in `tests/unit/tests_high_five_multi_dimensional.cpp`. This
+includes write/read cycles; checking all the generic edges cases, e.g. empty
+arrays and mismatching sizes; and checking non-reallocation.
+
+Read/Write cycles are implemented in two distinct checks. One for writing and
+another for reading. When checking writing we read with a "trusted"
+multi-dimensional array (a nested `std::vector`), and vice-versa when checking
+reading. This matters because certain bugs, like writing a column major array
+as if it were row-major can't be caught if one reads it back into a
+column-major array.
+
+Remember, `std::vector<bool>` is very different from all other `std::vector`s.
+
+Every container `template<class T> C;` should at least be checked with all of
+the following `T`s that are supported by the container: `bool`, `double`,
+`std::string`, `std::vector`, `std::array`. The reason is `bool` and
+`std::string` are special, `double` is just a POD, `std::vector` requires
+dynamic memory allocation and `std::array` is statically allocated.
+
+Similarly, each container should be put inside an `std::vector` and an
+`std::array`.
+
+#### Scalar Data Set
+Write-read cycles for scalar values should be implemented in
+`tests/unit/tests_high_five_scalar.cpp`.
+
+#### Data Types
+Unit-tests related to checking that `DataType` API, go in
+`tests/unit/tests_high_data_type.cpp`.
+
+#### Selections
+Anything selection related goes in `tests/unit/test_high_five_selection.cpp`.
+This includes things like `ElementSet` and `HyperSlab`.
+
+#### Strings
+Regular write-read cycles for strings are performed along with the other types,
+see above. This should cover compatibility of `std::string` with all
+containers. However, additional testing is required, e.g. character set,
+padding, fixed vs. variable length. These all go in
+`tests/unit/test_high_five_string.cpp`.
+
+#### Specific Tests For Optional Containers
+If containers, e.g. `Eigen::Matrix` require special checks those go in files
+called `tests/unit/test_high_five_*.cpp` where `*` is `eigen` for Eigen.
+
+#### Memory Layout Assumptions
+In HighFive we make assumptions about the memory layout of certain types. For
+example, we assume that
+```
+auto array = std::vector<std::array<double, 2>>(n);
+doube * ptr = (double*) array.data();
+```
+is a sensible thing to do. We assume similar about `bool` and
+`details::Boolean`. These types of tests go into
+`tests/unit/tests_high_five_memory_layout.cpp`.
+
+#### H5Easy
+Anything `H5Easy` related goes in files with the appropriate name.
+
+#### Everything Else
+What's left goes in `tests/unit/test_high_five_base.cpp`. This covers opening
+files, groups, dataset or attributes; checking certain pathological edge cases;
+etc.
diff --git a/tests/unit/create_traits.hpp b/tests/unit/create_traits.hpp
new file mode 100644
index 000000000..959fcdeb1
--- /dev/null
+++ b/tests/unit/create_traits.hpp
@@ -0,0 +1,70 @@
+#pragma once
+
+namespace HighFive {
+namespace testing {
+
+/// \brief Trait for `createAttribute`.
+///
+/// The point of these is to simplify testing. The typical issue is that we
+/// need to write the tests twice, one with `createDataSet` and then again with
+/// `createAttribute`. This trait allows us to inject this difference.
+struct AttributeCreateTraits {
+    using type = Attribute;
+
+    template <class Hi5>
+    static Attribute get(Hi5& hi5, const std::string& name) {
+        return hi5.getAttribute(name);
+    }
+
+
+    template <class Hi5, class Container>
+    static Attribute create(Hi5& hi5, const std::string& name, const Container& container) {
+        return hi5.createAttribute(name, container);
+    }
+
+    template <class Hi5>
+    static Attribute create(Hi5& hi5,
+                            const std::string& name,
+                            const DataSpace& dataspace,
+                            const DataType& datatype) {
+        return hi5.createAttribute(name, dataspace, datatype);
+    }
+
+    template <class T, class Hi5>
+    static Attribute create(Hi5& hi5, const std::string& name, const DataSpace& dataspace) {
+        auto datatype = create_datatype<T>();
+        return hi5.template createAttribute<T>(name, dataspace);
+    }
+};
+
+/// \brief Trait for `createDataSet`.
+struct DataSetCreateTraits {
+    using type = DataSet;
+
+    template <class Hi5>
+    static DataSet get(Hi5& hi5, const std::string& name) {
+        return hi5.getDataSet(name);
+    }
+
+    template <class Hi5, class Container>
+    static DataSet create(Hi5& hi5, const std::string& name, const Container& container) {
+        return hi5.createDataSet(name, container);
+    }
+
+    template <class Hi5>
+    static DataSet create(Hi5& hi5,
+                          const std::string& name,
+                          const DataSpace& dataspace,
+                          const DataType& datatype) {
+        return hi5.createDataSet(name, dataspace, datatype);
+    }
+
+    template <class T, class Hi5>
+    static DataSet create(Hi5& hi5, const std::string& name, const DataSpace& dataspace) {
+        auto datatype = create_datatype<T>();
+        return hi5.template createDataSet<T>(name, dataspace);
+    }
+};
+
+}  // namespace testing
+}  // namespace HighFive
diff --git a/tests/unit/data_generator.hpp b/tests/unit/data_generator.hpp
new file mode 100644
index 000000000..f0b0d2625
--- /dev/null
+++ b/tests/unit/data_generator.hpp
@@ -0,0 +1,463 @@
+#pragma once
+
+#include <limits>
+#include <numeric>
+#include <stdexcept>
+#include <type_traits>
+#include <vector>
+#include <array>
+
+#ifdef H5_USE_BOOST
+#include <boost/multi_array.hpp>
+#endif
+
+#include <highfive/bits/H5Inspector_misc.hpp>
+
+namespace HighFive {
+namespace testing {
+
+std::vector<size_t> lstrip(const std::vector<size_t>& indices, size_t n) {
+    std::vector<size_t> subindices(indices.size() - n);
+    for (size_t i = 0; i < subindices.size(); ++i) {
+        subindices[i] = indices[i + n];
+    }
+
+    return subindices;
+}
+
+size_t ravel(std::vector<size_t>& indices, const std::vector<size_t> dims) {
+    size_t rank = dims.size();
+    size_t linear_index = 0;
+    size_t ld = 1;
+    for (size_t kk = 0; kk < rank; ++kk) {
+        auto k = rank - 1 - kk;
+        linear_index += indices[k] * ld;
+        ld *= dims[k];
+    }
+
+    return linear_index;
+}
+
+std::vector<size_t> unravel(size_t flat_index, const std::vector<size_t> dims) {
+    size_t rank = dims.size();
+    size_t ld = 1;
+    std::vector<size_t> indices(rank);
+    for (size_t kk = 0; kk < rank; ++kk) {
+        auto k = rank - 1 - kk;
+        indices[k] = (flat_index / ld) % dims[k];
+        ld *= dims[k];
+    }
+
+    return indices;
+}
+
+static size_t flat_size(const std::vector<size_t>& dims) {
+    size_t n = 1;
+    for (auto d: dims) {
+        n *= d;
+    }
+
+    return n;
+}
+
+template <class Container, class = void>
+struct ContainerTraits;
+
+// -- Scalar basecases ---------------------------------------------------------
+template <class T>
+struct ScalarContainerTraits {
+    using container_type = T;
+    using base_type = T;
+
+    static void set(container_type& array, std::vector<size_t> /* indices */, base_type value) {
+        array = value;
+    }
+
+    static const base_type& get(const container_type& array, std::vector<size_t> /* indices */) {
+        return array;
+    }
+
+    static void assign(container_type& dst, const container_type& src) {
+        dst = src;
+    }
+
+    static container_type allocate(const std::vector<size_t>& /* dims */) {
+        return container_type{};
+    }
+
+    static void sanitize_dims(std::vector<size_t>& /* dims */, size_t /* axis */) {}
+};
+
+template <class T>
+struct ContainerTraits<T, typename std::enable_if<std::is_floating_point<T>::value>::type>
+    : public ScalarContainerTraits<T> {};
+
+template <class T>
+struct ContainerTraits<T, typename std::enable_if<std::is_integral<T>::value>::type>
+    : public ScalarContainerTraits<T> {};
+
+template <>
+struct ContainerTraits<std::string>: public ScalarContainerTraits<std::string> {};
+
+// -- STL ----------------------------------------------------------------------
+template <>
+struct ContainerTraits<std::vector<bool>> {
+    using container_type = std::vector<bool>;
+    using value_type = bool;
+    using base_type = bool;
+
+    static void set(container_type& array,
+                    const std::vector<size_t>& indices,
+                    const base_type& value) {
+        array[indices[0]] = value;
+    }
+
+    static base_type get(const container_type& array, const std::vector<size_t>& indices) {
+        return array[indices[0]];
+    }
+
+    static void assign(container_type& dst, const container_type& src) {
+        dst = src;
+    }
+
+    static container_type allocate(const std::vector<size_t>& dims) {
+        container_type array(dims[0]);
+        return array;
+    }
+
+    static void sanitize_dims(std::vector<size_t>& dims, size_t axis) {
+        ContainerTraits<value_type>::sanitize_dims(dims, axis + 1);
+    }
+};
+
+template <class Container, class ValueType = typename Container::value_type>
+struct STLLikeContainerTraits {
+    using container_type = Container;
+    using value_type = ValueType;
+    using base_type = typename ContainerTraits<value_type>::base_type;
+
+    static void set(container_type& array,
+                    const std::vector<size_t>& indices,
+                    const base_type& value) {
+        return ContainerTraits<value_type>::set(array[indices[0]], lstrip(indices, 1), value);
+    }
+
+    static base_type get(const container_type& array, const std::vector<size_t>& indices) {
+        return ContainerTraits<value_type>::get(array[indices[0]], lstrip(indices, 1));
+    }
+
+    static void assign(container_type& dst, const container_type& src) {
+        dst = src;
+    }
+
+    static container_type allocate(const std::vector<size_t>& dims) {
+        container_type array(dims[0]);
+        for (size_t i = 0; i < dims[0]; ++i) {
+            auto value = ContainerTraits<value_type>::allocate(lstrip(dims, 1));
+            ContainerTraits<value_type>::assign(array[i], value);
+        }
+
+        return array;
+    }
+
+    static void sanitize_dims(std::vector<size_t>& dims, size_t axis) {
+        ContainerTraits<value_type>::sanitize_dims(dims, axis + 1);
+    }
+};
+
+template <class T>
+struct ContainerTraits<std::vector<T>>: public STLLikeContainerTraits<std::vector<T>> {};
+
+template <class T, size_t N>
+struct ContainerTraits<std::array<T, N>>: public STLLikeContainerTraits<std::array<T, N>> {
+  private:
+    using super = STLLikeContainerTraits<std::array<T, N>>;
+
+  public:
+    using container_type = typename super::container_type;
+    using base_type = typename super::base_type;
+    using value_type = typename super::value_type;
+
+  public:
+    static container_type allocate(const std::vector<size_t>& dims) {
+        if (N != dims[0]) {
+            throw std::runtime_error("broken logic: static and runtime size don't match.");
+        }
+
+        container_type array;
+        for (size_t i = 0; i < dims[0]; ++i) {
+            auto value = ContainerTraits<value_type>::allocate(lstrip(dims, 1));
+            ContainerTraits<value_type>::assign(array[i], value);
+        }
+
+        return array;
+    }
+
+    static void sanitize_dims(std::vector<size_t>& dims, size_t axis) {
+        dims[axis] = N;
+        ContainerTraits<value_type>::sanitize_dims(dims, axis + 1);
+    }
+};
+
+// -- Boost  -------------------------------------------------------------------
+#ifdef H5_USE_BOOST
+template <class T, size_t n>
+struct ContainerTraits<boost::multi_array<T, n>> {
+    using container_type = typename boost::multi_array<T, n>;
+    using value_type = T;
+    using base_type = typename ContainerTraits<value_type>::base_type;
+
+    static void set(container_type& array,
+                    const std::vector<size_t>& indices,
+                    const base_type& value) {
+        auto i = std::vector<size_t>(indices.begin(), indices.begin() + n);
+        return ContainerTraits<value_type>::set(array(i), lstrip(indices, n), value);
+    }
+
+    static base_type get(const container_type& array, const std::vector<size_t>& indices) {
+        auto i = std::vector<size_t>(indices.begin(), indices.begin() + n);
+        return ContainerTraits<value_type>::get(array(i), lstrip(indices, n));
+    }
+
+    static void assign(container_type& dst, const container_type& src) {
+        auto const* const shape = src.shape();
+        dst.resize(std::vector<size_t>(shape, shape + n));
+        dst = src;
+    }
+
+    static container_type allocate(const std::vector<size_t>& dims) {
+        auto local_dims = std::vector<size_t>(dims.begin(), dims.begin() + n);
+        container_type array(local_dims);
+
+        size_t n_elements = flat_size(local_dims);
+        for (size_t i = 0; i < n_elements; ++i) {
+            auto element = ContainerTraits<value_type>::allocate(lstrip(dims, n));
+            set(array, unravel(i, local_dims), element);
+        }
+
+        return array;
+    }
+
+    static void sanitize_dims(std::vector<size_t>& dims, size_t axis) {
+        ContainerTraits<value_type>::sanitize_dims(dims, axis + n);
+    }
+};
+
+template <class T>
+struct ContainerTraits<boost::numeric::ublas::matrix<T>> {
+    using container_type = typename boost::numeric::ublas::matrix<T>;
+    using value_type = T;
+    using base_type = typename ContainerTraits<value_type>::base_type;
+
+    static void set(container_type& array,
+                    const std::vector<size_t>& indices,
+                    const base_type& value) {
+        auto i = indices[0];
+        auto j = indices[1];
+        return ContainerTraits<value_type>::set(array(i, j), lstrip(indices, 2), value);
+    }
+
+    static base_type get(const container_type& array, const std::vector<size_t>& indices) {
+        auto i = indices[0];
+        auto j = indices[1];
+        return ContainerTraits<value_type>::get(array(i, j), lstrip(indices, 2));
+    }
+
+    static void assign(container_type& dst, const container_type& src) {
+        dst = src;
+    }
+
+    static container_type allocate(const std::vector<size_t>& dims) {
+        auto local_dims = std::vector<size_t>(dims.begin(), dims.begin() + 2);
+        container_type array(local_dims[0], local_dims[1]);
+
+        size_t n_elements = flat_size(local_dims);
+        for (size_t i = 0; i < n_elements; ++i) {
+            auto indices = unravel(i, local_dims);
+            auto element = ContainerTraits<value_type>::allocate(lstrip(dims, 2));
+
+            ContainerTraits<value_type>::assign(array(indices[0], indices[1]), element);
+        }
+
+        return array;
+    }
+
+    static void sanitize_dims(std::vector<size_t>& dims, size_t axis) {
+        ContainerTraits<value_type>::sanitize_dims(dims, axis + 2);
+    }
+};
+
+#endif
+
+template <class T, class C>
+T copy(const C& src, const std::vector<size_t>& dims) {
+    auto dst = ContainerTraits<T>::allocate(dims);
+    for (size_t i = 0; i < flat_size(dims); ++i) {
+        auto indices = unravel(i, dims);
+        ContainerTraits<T>::set(dst, indices, ContainerTraits<C>::get(src, indices));
+    }
+
+    return dst;
+}
+
+template <class T>
+T default_real_value(const std::vector<size_t>& indices, T shift, T base, T factor) {
+    auto value = T(0);
+
+    auto isum = std::accumulate(indices.begin(), indices.end(), size_t(0));
+    auto sign = (std::is_signed<T>::value) && (isum % 2 == 1) ? T(-1) : T(1);
+
+    for (size_t k = 0; k < indices.size(); ++k) {
+        value += T(indices[k]) * T(std::pow(shift, T(k))) * base;
+    }
+
+    return sign * value * factor;
+}
+
+std::vector<std::string> ascii_alphabet = {"a", "b", "c", "d", "e", "f"};
+
+std::string default_string(size_t offset, size_t length, const std::vector<std::string>& alphabet) {
+    std::string s = "";
+    for (size_t k = 0; k < length; ++k) {
+        s += alphabet[(offset + k) % alphabet.size()];
+    }
+
+    return s;
+}
+
+std::string default_fixed_length_ascii_string(const std::vector<size_t>& indices, size_t length) {
+    auto isum = std::accumulate(indices.begin(), indices.end(), size_t(0));
+    return default_string(isum, length, ascii_alphabet);
+}
+
+std::string default_variable_length_ascii_string(const std::vector<size_t>& indices) {
+    auto isum = std::accumulate(indices.begin(), indices.end(), size_t(0));
+    return default_string(isum, isum, ascii_alphabet);
+}
+
+template <class T, class = void>
+struct DefaultValues;
+
+template <class T>
+struct DefaultValues<T, typename std::enable_if<std::is_floating_point<T>::value>::type> {
+    T operator()(const std::vector<size_t>& indices) const {
+        auto eps = std::numeric_limits<T>::epsilon();
+        return default_real_value(indices, T(100.0), T(0.01), T(1.0) + T(8) * eps);
+    }
+};
+
+template <class T>
+struct DefaultValues<T, typename std::enable_if<std::is_integral<T>::value>::type> {
+    T operator()(const std::vector<size_t>& indices) const {
+        return default_real_value(indices, T(100), T(1), T(1));
+    }
+};
+
+template <>
+struct DefaultValues<char> {
+    char operator()(const std::vector<size_t>& indices) const {
+        auto isum = std::accumulate(indices.begin(), indices.end(), size_t(0));
+        return char(isum % size_t(std::numeric_limits<char>::max));
+    }
+};
+
+template <>
+struct DefaultValues<unsigned char> {
+    unsigned char operator()(const std::vector<size_t>& indices) const {
+        auto isum = std::accumulate(indices.begin(), indices.end(), size_t(0));
+        return (unsigned char) (isum % size_t(std::numeric_limits<unsigned char>::max));
+    }
+};
+
+template <>
+struct DefaultValues<std::string> {
+    std::string operator()(const std::vector<size_t>& indices) const {
+        return default_variable_length_ascii_string(indices);
+    }
+};
+
+template <>
+struct DefaultValues<bool> {
+    bool operator()(const std::vector<size_t>& indices) const {
+        auto isum = std::accumulate(indices.begin(), indices.end(), size_t(0));
+        return (isum % 2) == 0;
+    }
+};
+
+template <class T, size_t N>
+struct MultiDimVector {
+    using type = std::vector<typename MultiDimVector<T, N - 1>::type>;
+};
+
+template <class T>
+struct MultiDimVector<T, 0> {
+    using type = T;
+};
+
+template <class Container>
+class DataGenerator {
+  public:
+    constexpr static size_t rank = details::inspector<Container>::recursive_ndim;
+    using traits = ContainerTraits<Container>;
+    using base_type = typename traits::base_type;
+    using container_type = Container;
+
+  public:
+    static container_type allocate(const std::vector<size_t>& dims) {
+        return traits::allocate(dims);
+    }
+
+    template <class F>
+    static container_type create(const std::vector<size_t>& dims, F f) {
+        std::cout << "allocate " << std::endl;
+        auto array = allocate(dims);
+        std::cout << "initialize " << std::endl;
+        initialize(array, dims, f);
+
+        return array;
+    }
+
+    static container_type create(const std::vector<size_t>& dims) {
+        return create(dims, DefaultValues<typename traits::base_type>());
+    }
+
+    static std::vector<size_t> default_dims() {
+        using difference_type = std::vector<size_t>::difference_type;
+        std::vector<size_t> oversized{2, 3, 5, 7, 2, 3, 5, 7};
+        std::vector<size_t> dims(oversized.begin(), oversized.begin() + difference_type(rank));
+        ContainerTraits<Container>::sanitize_dims(dims, /* axis = */ 0);
+
+        return dims;
+    }
+
+    static void sanitize_dims(std::vector<size_t>& dims) {
+        ContainerTraits<Container>::sanitize_dims(dims, /* axis = */ 0);
+    }
+
+  private:
+    template <class C, class F>
+    static void initialize(C& array, const std::vector<size_t>& dims, F f) {
+        std::vector<size_t> indices(dims.size());
+        initialize(array, dims, indices, 0, f);
+    }
+
+    template <class C, class F>
+    static void initialize(C& array,
+                           const std::vector<size_t>& dims,
+                           std::vector<size_t>& indices,
+                           size_t axis,
+                           F f) {
+        if (axis == indices.size()) {
+            auto value = f(indices);
+            traits::set(array, indices, value);
+        } else {
+            for (size_t i = 0; i < dims[axis]; ++i) {
+                indices[axis] = i;
+                initialize(array, dims, indices, axis + 1, f);
+            }
+        }
+    }
+};
+
+}  // namespace testing
+}  // namespace HighFive
diff --git a/tests/unit/supported_types.hpp b/tests/unit/supported_types.hpp
new file mode 100644
index 000000000..f708303b1
--- /dev/null
+++ b/tests/unit/supported_types.hpp
@@ -0,0 +1,108 @@
+
+#pragma once
+
+#include <type_traits>
+#include <vector>
+#include <array>
+#include <tuple>
+
+#ifdef H5_USE_BOOST
+#include <boost/multi_array.hpp>
+#endif
+
+namespace HighFive {
+namespace testing {
+
+struct type_identity {
+    template <class T>
+    using type = T;
+};
+
+template <class C = type_identity>
+struct STDVector {
+    template <class T>
+    using type = std::vector<typename C::template type<T>>;
+};
+
+template <size_t n, class C = type_identity>
+struct STDArray {
+    template <class T>
+    using type = std::array<typename C::template type<T>, n>;
+};
+
+#ifdef H5_USE_BOOST
+template <size_t n, class C = type_identity>
+struct BoostMultiArray {
+    template <class T>
+    using type = boost::multi_array<typename C::template type<T>, 4>;
+};
+
+template <class C = type_identity>
+struct BoostUblasMatrix {
+    template <class T>
+    using type = boost::numeric::ublas::matrix<typename C::template type<T>>;
+};
+#endif
+
+template <class C, class Tuple>
+struct ContainerProduct;
+
+template <class C, class... ScalarTypes>
+struct ContainerProduct<C, std::tuple<ScalarTypes...>> {
+    using type = std::tuple<typename C::template type<ScalarTypes>...>;
+};
+
+template <class... Tuples>
+struct ConcatenateTuples;
+
+template <class... Args1, class... Args2, class... Tuples>
+struct ConcatenateTuples<std::tuple<Args1...>, std::tuple<Args2...>, Tuples...> {
+    using type = typename ConcatenateTuples<std::tuple<Args1..., Args2...>, Tuples...>::type;
+};
+
+template <class... Args1>
+struct ConcatenateTuples<std::tuple<Args1...>> {
+    using type = std::tuple<Args1...>;
+};
+
+// clang-format off
+using numeric_scalar_types = std::tuple<
+    int,
+    unsigned int,
+    long,
+    unsigned long,
+    unsigned char,
+    char,
+    float,
+    double,
+    long long,
+    unsigned long long
+>;
+
+using scalar_types = typename ConcatenateTuples<numeric_scalar_types, std::tuple<bool, std::string>>::type;
+using scalar_types_boost = typename ConcatenateTuples<numeric_scalar_types, std::tuple<bool>>::type;
+
+using supported_array_types = typename ConcatenateTuples<
+#ifdef H5_USE_BOOST
+  typename ContainerProduct<BoostMultiArray<3>, scalar_types_boost>::type,
+  typename ContainerProduct<STDVector<BoostMultiArray<3>>, scalar_types_boost>::type,
+  typename ContainerProduct<STDArray<5, BoostMultiArray<3>>, scalar_types_boost>::type,
+
+  typename ContainerProduct<BoostUblasMatrix<>, scalar_types_boost>::type,
+  typename ContainerProduct<STDVector<BoostUblasMatrix<>>, scalar_types_boost>::type,
+  typename ContainerProduct<STDArray<5, BoostUblasMatrix<>>, scalar_types_boost>::type,
+#endif
+  typename ContainerProduct<STDVector<>, scalar_types>::type,
+  typename ContainerProduct<STDVector<STDVector<>>, scalar_types>::type,
+  typename ContainerProduct<STDVector<STDVector<STDVector<>>>, scalar_types>::type,
+  typename ContainerProduct<STDVector<STDVector<STDVector<STDVector<>>>>, scalar_types>::type,
+  typename ContainerProduct<STDArray<3>, scalar_types>::type,
+  typename ContainerProduct<STDArray<7, STDArray<5>>, scalar_types>::type,
+  typename ContainerProduct<STDVector<STDArray<5>>, scalar_types>::type,
+  typename ContainerProduct<STDArray<7, STDVector<>>, scalar_types>::type
+>::type;
+
+// clang-format on
+
+}  // namespace testing
+}  // namespace HighFive
diff --git a/tests/unit/test_all_types.cpp b/tests/unit/test_all_types.cpp
index 23c8a27b3..e772fd1d7 100644
--- a/tests/unit/test_all_types.cpp
+++ b/tests/unit/test_all_types.cpp
@@ -7,11 +7,16 @@
  *
  */
 #include <string>
+#include <sstream>
 
 #include <catch2/catch_template_test_macros.hpp>
 
 #include <highfive/highfive.hpp>
+#include <type_traits>
 #include "tests_high_five.hpp"
+#include "data_generator.hpp"
+#include "create_traits.hpp"
+#include "supported_types.hpp"
 
 using namespace HighFive;
 
@@ -238,3 +243,244 @@ TEMPLATE_PRODUCT_TEST_CASE("Scalar in std::vector<std::byte>", "[Types]", std::v
     }
 }
 #endif
+
+template <class T, class = void>
+struct DiffMessageTrait;
+
+template <class T>
+struct DiffMessageTrait<T, typename std::enable_if<std::is_floating_point<T>::value>::type> {
+    static std::string diff(T a, T b) {
+        std::stringstream sstream;
+        sstream << std::scientific << " delta: " << a - b;
+        return sstream.str();
+    }
+};
+
+template <class T>
+struct DiffMessageTrait<T, typename std::enable_if<!std::is_floating_point<T>::value>::type> {
+    static std::string diff(T /* a */, T /* b */) {
+        return "";
+    }
+};
+
+template <class T>
+std::string diff_message(T a, T b) {
+    return DiffMessageTrait<T>::diff(a, b);
+}
+
+template <class Actual, class Expected, class Comp>
+void compare_arrays(const Actual& actual,
+                    const Expected& expected,
+                    const std::vector<size_t>& dims,
+                    Comp comp) {
+    using actual_trait = testing::ContainerTraits<Actual>;
+    using expected_trait = testing::ContainerTraits<Expected>;
+    using base_type = typename actual_trait::base_type;
+
+    auto n = testing::flat_size(dims);
+
+    for (size_t i = 0; i < n; ++i) {
+        auto indices = testing::unravel(i, dims);
+        base_type actual_value = actual_trait::get(actual, indices);
+        base_type expected_value = expected_trait::get(expected, indices);
+        auto c = comp(actual_value, expected_value);
+        if (!c) {
+            std::stringstream sstream;
+            sstream << std::scientific << "i = " << i << ": " << actual_value
+                    << " != " << expected_value << diff_message(actual_value, expected_value);
+            INFO(sstream.str());
+        }
+        REQUIRE(c);
+    }
+}
+
+template <class Actual, class Expected>
+void compare_arrays(const Actual& actual,
+                    const Expected& expected,
+                    const std::vector<size_t>& dims) {
+    using base_type = typename testing::ContainerTraits<Actual>::base_type;
+    compare_arrays(expected, actual, dims, [](base_type a, base_type b) { return a == b; });
+}
+
+template <class Container, class Expected, class Obj>
+void check_read_auto(const Expected& expected, const std::vector<size_t>& dims, const Obj& obj) {
+    compare_arrays(obj.template read<Container>(), expected, dims);
+}
+
+template <class Container, class Expected, class Obj>
+void check_read_preallocated(const Expected& expected,
+                             const std::vector<size_t>& dims,
+                             const Obj& obj) {
+    auto actual = testing::DataGenerator<Container>::allocate(dims);
+    obj.read(actual);
+
+    compare_arrays(actual, expected, dims);
+}
+
+template <class Container>
+void check_read_regular(const std::string& file_name, const std::vector<size_t>& dims) {
+    using traits = testing::DataGenerator<Container>;
+    using base_type = typename traits::base_type;
+    using reference_type = typename testing::MultiDimVector<base_type, traits::rank>::type;
+
+    auto file = File(file_name, File::Truncate);
+    auto expected = testing::copy<reference_type>(traits::create(dims), dims);
+
+    auto dataspace = DataSpace(dims);
+    auto attr = testing::AttributeCreateTraits::create<base_type>(file, "dset", dataspace);
+    attr.write(expected);
+
+    auto dset = testing::DataSetCreateTraits::create<base_type>(file, "attr", dataspace);
+    dset.write(expected);
+
+
+    SECTION("dset.read<Container>()") {
+        check_read_auto<Container>(expected, dims, dset);
+    }
+
+    SECTION("dset.read(values)") {
+        check_read_preallocated<Container>(expected, dims, dset);
+    }
+
+    SECTION("attr.read<Container>()") {
+        check_read_auto<Container>(expected, dims, attr);
+    }
+
+    SECTION("attr.read(values)") {
+        check_read_preallocated<Container>(expected, dims, attr);
+    }
+}
+
+template <class Container>
+void check_read_regular() {
+    const std::string file_name("rw_read_regular" + typeNameHelper<Container>() + ".h5");
+    auto dims = testing::DataGenerator<Container>::default_dims();
+
+    check_read_regular<Container>(file_name, dims);
+}
+
+TEMPLATE_LIST_TEST_CASE("TestReadRegular", "[read]", testing::supported_array_types) {
+    check_read_regular<TestType>();
+}
+
+template <class Container, class Write>
+void check_writing(const std::vector<size_t>& dims, Write write) {
+    using traits = testing::DataGenerator<Container>;
+    using base_type = typename traits::base_type;
+    using reference_type = typename testing::MultiDimVector<base_type, traits::rank>::type;
+
+    auto values = testing::DataGenerator<Container>::create(dims);
+    auto expected = testing::copy<reference_type>(values, dims);
+
+    auto obj = write(values);
+
+    auto actual = testing::DataGenerator<reference_type>::allocate(dims);
+    obj.read(actual);
+
+    compare_arrays(actual, expected, dims);
+}
+
+template <class CreateTraits, class Container>
+void check_write_auto(File& file, const std::string& name, const std::vector<size_t>& dims) {
+    auto write_auto = [&](const Container& values) {
+        return CreateTraits::create(file, "auto_" + name, values);
+    };
+
+    check_writing<Container>(dims, write_auto);
+}
+
+template <class CreateTraits, class Container>
+void check_write_deduce_type(File& file, const std::string& name, const std::vector<size_t>& dims) {
+    auto write_two_phase_auto = [&](const Container& values) {
+        using traits = testing::ContainerTraits<Container>;
+        auto dataspace = DataSpace(dims);
+        auto h5 = CreateTraits::template create<typename traits::base_type>(file,
+                                                                            "two_phase_auto" + name,
+                                                                            dataspace);
+        h5.write(values);
+        return h5;
+    };
+    check_writing<Container>(dims, write_two_phase_auto);
+}
+
+template <class CreateTraits, class Container>
+void check_write_manual(File& file, const std::string& name, const std::vector<size_t>& dims) {
+    auto write_two_phase = [&](const Container& values) {
+        using traits = testing::ContainerTraits<Container>;
+        auto datatype = create_datatype<typename traits::base_type>();
+        auto dataspace = DataSpace(dims);
+        auto h5 = CreateTraits::create(file, "two_phase_" + name, dataspace, datatype);
+        h5.write(values);
+        return h5;
+    };
+    check_writing<Container>(dims, write_two_phase);
+}
+
+template <class Container>
+void check_write_regular(const std::string& file_name, const std::vector<size_t>& dims) {
+    auto file = File(file_name, File::Truncate);
+
+    SECTION("createDataSet(name, container)") {
+        check_write_auto<testing::DataSetCreateTraits, Container>(file, "dset", dims);
+    }
+
+    SECTION("createDataSet(name, container)") {
+        check_write_deduce_type<testing::DataSetCreateTraits, Container>(file, "dset", dims);
+    }
+
+    SECTION("createDataSet(name, container)") {
+        check_write_manual<testing::DataSetCreateTraits, Container>(file, "dset", dims);
+    }
+
+    SECTION("createAttribute(name, container)") {
+        check_write_auto<testing::AttributeCreateTraits, Container>(file, "attr", dims);
+    }
+
+    SECTION("createAttribute(name, container)") {
+        check_write_deduce_type<testing::AttributeCreateTraits, Container>(file, "attr", dims);
+    }
+
+    SECTION("createAttribute(name, container)") {
+        check_write_manual<testing::AttributeCreateTraits, Container>(file, "attr", dims);
+    }
+}
+
+template <class Container>
+void check_write_regular() {
+    std::string file_name("rw_write_regular" + typeNameHelper<Container>() + ".h5");
+    auto dims = testing::DataGenerator<Container>::default_dims();
+    check_write_regular<Container>(file_name, dims);
+}
+
+TEMPLATE_LIST_TEST_CASE("TestWriteRegularSTDVector", "[write]", testing::supported_array_types) {
+    check_write_regular<TestType>();
+}
+
+TEST_CASE("DataGeneratorDefaultDims", "[internal]") {
+    SECTION("std::array") {
+        auto dims = testing::DataGenerator<std::array<double, 3>>::default_dims();
+        REQUIRE(dims.size() == 1);
+        CHECK(dims[0] == 3);
+    }
+
+    SECTION("std::vector") {
+        auto dims = testing::DataGenerator<std::vector<double>>::default_dims();
+        REQUIRE(dims.size() == 1);
+        CHECK(dims[0] > 0);
+    }
+
+    SECTION("std::vector<std::vector>") {
+        auto dims = testing::DataGenerator<std::vector<std::vector<double>>>::default_dims();
+        REQUIRE(dims.size() == 2);
+        CHECK(dims[0] * dims[1] > 0);
+    }
+}
+
+TEST_CASE("ravel", "[internal]") {
+    std::vector<size_t> dims = {2, 4, 5};
+    std::vector<size_t> indices = {1, 2, 3};
+    size_t flat_index = indices[2] + dims[2] * (indices[1] + dims[1] * indices[0]);
+
+    CHECK(flat_index == testing::ravel(indices, dims));
+    CHECK(indices == testing::unravel(flat_index, dims));
+}
diff --git a/tests/unit/tests_high_five_multi_dims.cpp b/tests/unit/tests_high_five_multi_dims.cpp
index 442f1c9cc..08fbea9ce 100644
--- a/tests/unit/tests_high_five_multi_dims.cpp
+++ b/tests/unit/tests_high_five_multi_dims.cpp
@@ -80,11 +80,11 @@ void readWriteArrayTest() {
     typename std::array<T, 1> tooSmall;
     CHECK_THROWS_AS(dataset.read(tooSmall), DataSpaceException);
 }
+
 TEMPLATE_LIST_TEST_CASE("readWriteArray", "[template]", numerical_test_types) {
     readWriteArrayTest<TestType>();
 }
 
-
 template <typename T, typename VectorSubT>
 void readWriteVectorNDTest(std::vector<VectorSubT>& ndvec, const std::vector<size_t>& dims) {
     fillVec(ndvec, dims, ContentGenerate<T>());