diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.cpp b/source/adios2/engine/hdf5/HDF5ReaderP.cpp index 2052858156..65f796d1f6 100644 --- a/source/adios2/engine/hdf5/HDF5ReaderP.cpp +++ b/source/adios2/engine/hdf5/HDF5ReaderP.cpp @@ -372,6 +372,7 @@ void HDF5ReaderP::DoClose(const int transportIndex) m_H5File.Close(); } +size_t HDF5ReaderP::DoSteps() const { return m_H5File.GetAdiosStep(); } } // end namespace engine } // end namespace core } // end namespace adios2 diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.h b/source/adios2/engine/hdf5/HDF5ReaderP.h index 9208fe6c99..244d5f3c74 100644 --- a/source/adios2/engine/hdf5/HDF5ReaderP.h +++ b/source/adios2/engine/hdf5/HDF5ReaderP.h @@ -97,6 +97,8 @@ class HDF5ReaderP : public Engine void UseHDFRead(Variable &variable, T *values, hid_t h5Type); std::vector m_DeferredStack; + + size_t DoSteps() const final; }; } // end namespace engine diff --git a/source/adios2/engine/hdf5/HDF5WriterP.cpp b/source/adios2/engine/hdf5/HDF5WriterP.cpp index 43d4312d7a..f7cc30495c 100644 --- a/source/adios2/engine/hdf5/HDF5WriterP.cpp +++ b/source/adios2/engine/hdf5/HDF5WriterP.cpp @@ -74,13 +74,25 @@ void HDF5WriterP::Init() { // is a file with .bp ending std::string updatedName = m_Name.substr(0, wpos) + suffix; - m_H5File.Init(updatedName, m_Comm, true); + if (m_OpenMode == Mode::Append) + m_H5File.Append(updatedName, m_Comm); + else + m_H5File.Init(updatedName, m_Comm, true); } else { - m_H5File.Init(m_Name, m_Comm, true); + if (m_OpenMode == Mode::Append) + m_H5File.Append(m_Name, m_Comm); + else + m_H5File.Init(m_Name, m_Comm, true); } m_H5File.ParseParameters(m_IO); + + if (m_OpenMode == Mode::Append) + { + m_H5File.ReadAttrToIO(m_IO); + m_H5File.ReadAllVariables(m_IO); + } #endif } diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp index 8a13a35edf..bd4071fa9b 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp @@ -135,6 +135,52 @@ void HDF5Common::ParseParameters(core::IO &io) } } +void HDF5Common::Append(const std::string &name, helper::Comm const &comm) +{ + m_PropertyListId = H5Pcreate(H5P_FILE_ACCESS); + + if (MPI_API const *mpi = GetHDF5Common_MPI_API()) + { + if (mpi && mpi->init(comm, m_PropertyListId, &m_CommRank, &m_CommSize)) + { + m_MPI = mpi; + } + } + + m_FileId = H5Fopen(name.c_str(), H5F_ACC_RDWR, m_PropertyListId); + H5Pclose(m_PropertyListId); + + std::string ts0; + StaticGetAdiosStepString(ts0, 0); + + if (m_FileId >= 0) + { + if (H5Lexists(m_FileId, ts0.c_str(), H5P_DEFAULT) != 0) + { + m_IsGeneratedByAdios = true; + } + if (!m_IsGeneratedByAdios) + throw std::ios_base::failure( + "HDF5Engine Append error. Likely no such file." + name); + + GetNumAdiosSteps(); // read how many steps exists in this file + + if (0 == m_NumAdiosSteps) + throw std::ios_base::failure( + "HDF5Engine Append error. No valid steps found in " + name); + if (1 == m_NumAdiosSteps) + m_GroupId = H5Gopen(m_FileId, ts0.c_str(), H5P_DEFAULT); + else + SetAdiosStep(m_NumAdiosSteps - 1); + + m_WriteMode = true; + Advance(); + } + else + throw std::ios_base::failure( + "HDF5Engine Append error. Likely no such file." + name); +} + void HDF5Common::Init(const std::string &name, helper::Comm const &comm, bool toWrite) { @@ -204,9 +250,13 @@ void HDF5Common::WriteAdiosSteps() } hid_t s = H5Screate(H5S_SCALAR); - hid_t attr = - H5Acreate(m_FileId, ATTRNAME_NUM_STEPS.c_str(), - /*"NumSteps",*/ H5T_NATIVE_UINT, s, H5P_DEFAULT, H5P_DEFAULT); + hid_t attr = H5Aexists(m_FileId, ATTRNAME_NUM_STEPS.c_str()); + if (0 == attr) + attr = H5Acreate(m_FileId, ATTRNAME_NUM_STEPS.c_str(), H5T_NATIVE_UINT, + s, H5P_DEFAULT, H5P_DEFAULT); + else + attr = H5Aopen(m_FileId, ATTRNAME_NUM_STEPS.c_str(), H5P_DEFAULT); + unsigned int totalAdiosSteps = m_CurrentAdiosStep + 1; if (m_GroupId < 0) @@ -220,6 +270,8 @@ void HDF5Common::WriteAdiosSteps() H5Aclose(attr); } +unsigned int HDF5Common::GetAdiosStep() const { return m_NumAdiosSteps; } + unsigned int HDF5Common::GetNumAdiosSteps() { if (m_WriteMode) diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.h b/source/adios2/toolkit/interop/hdf5/HDF5Common.h index 39bdb44398..296e997115 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.h +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.h @@ -127,6 +127,7 @@ class HDF5Common void ParseParameters(core::IO &io); void Init(const std::string &name, helper::Comm const &comm, bool toWrite); + void Append(const std::string &name, helper::Comm const &comm); template void Write(core::Variable &variable, const T *values); @@ -172,6 +173,7 @@ class HDF5Common void SetAdiosStep(int ts); unsigned int GetNumAdiosSteps(); + unsigned int GetAdiosStep() const; void WriteAdiosSteps(); void ReadVariables(unsigned int ts, core::IO &io); diff --git a/testing/adios2/engine/hdf5/CMakeLists.txt b/testing/adios2/engine/hdf5/CMakeLists.txt index ca96432f04..1746648163 100644 --- a/testing/adios2/engine/hdf5/CMakeLists.txt +++ b/testing/adios2/engine/hdf5/CMakeLists.txt @@ -23,6 +23,10 @@ gtest_add_tests_helper(WriteMemorySelectionRead ${hdf5_mpi} HDF5 Engine.HDF5. "" ) +gtest_add_tests_helper(Append ${hdf5_mpi} + HDF5 Engine.HDF5. "" +) + gtest_add_tests_helper(NativeHDF5WriteRead ${hdf5_mpi} "" Engine.HDF5. "") if(HDF5_C_INCLUDE_DIRS) target_include_directories(Test.Engine.HDF5.NativeHDF5WriteRead${hdf5_sfx} diff --git a/testing/adios2/engine/hdf5/TestHDF5Append.cpp b/testing/adios2/engine/hdf5/TestHDF5Append.cpp new file mode 100644 index 0000000000..caec6045ae --- /dev/null +++ b/testing/adios2/engine/hdf5/TestHDF5Append.cpp @@ -0,0 +1,391 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ +#include +#include + +#include +#include + +#include + +#include + +#include "../SmallTestData.h" + +std::string engineName; // comes from command line + +class AppendTimeStepTest : public ::testing::Test +{ +public: + AppendTimeStepTest() = default; + + SmallTestData m_TestData; +}; + +//****************************************************************************** +// 1D 1x8 test data +//****************************************************************************** + +// ADIOS2 HDF5 write, then append, then read back +TEST_F(AppendTimeStepTest, ADIOS2HDF5WriteAppendRead) +{ + // Each process would write a 1x8 array and all processes would + // form a mpiSize * Nx 1D array + std::string fname = "appendTest.h5"; + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 8; + + // Number of steps + const std::size_t NSteps = 3; + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + + // Write test data using HDF5 engine + { + adios2::IO io = adios.DeclareIO("TestIO"); + + // Declare 1D variables (NumOfProcesses * Nx) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + adios2::Dims shape{static_cast(Nx * mpiSize)}; + adios2::Dims start{static_cast(Nx * mpiRank)}; + adios2::Dims count{static_cast(Nx)}; + + auto var_iString = io.DefineVariable("iString"); + auto var_i8 = io.DefineVariable("i8", shape, start, count); + auto var_i16 = + io.DefineVariable("i16", shape, start, count); + auto var_i32 = + io.DefineVariable("i32", shape, start, count); + auto var_i64 = + io.DefineVariable("i64", shape, start, count); + auto var_u8 = io.DefineVariable("u8", shape, start, count); + auto var_u16 = + io.DefineVariable("u16", shape, start, count); + auto var_u32 = + io.DefineVariable("u32", shape, start, count); + auto var_u64 = + io.DefineVariable("u64", shape, start, count); + auto var_r32 = io.DefineVariable("r32", shape, start, count); + auto var_r64 = + io.DefineVariable("r64", shape, start, count); + } + + if (!engineName.empty()) + io.SetEngine(engineName); + else + io.SetEngine("HDF5"); + + io.AddTransport("file"); + + adios2::Engine engine = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); + + // Retrieve the variables that previously went out of scope + auto var_iString = io.InquireVariable("iString"); + auto var_i8 = io.InquireVariable("i8"); + auto var_i16 = io.InquireVariable("i16"); + auto var_i32 = io.InquireVariable("i32"); + auto var_i64 = io.InquireVariable("i64"); + auto var_u8 = io.InquireVariable("u8"); + auto var_u16 = io.InquireVariable("u16"); + auto var_u32 = io.InquireVariable("u32"); + auto var_u64 = io.InquireVariable("u64"); + auto var_r32 = io.InquireVariable("r32"); + auto var_r64 = io.InquireVariable("r64"); + + // Make a 1D selection to describe the local dimensions of the + // variable we write and its offsets in the global spaces + adios2::Box sel({mpiRank * Nx}, {Nx}); + + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + // Write each one + // fill in the variable with values from starting index to + // starting index + count + engine.BeginStep(); + engine.Put(var_iString, currentTestData.S1); + engine.Put(var_i8, currentTestData.I8.data()); + engine.Put(var_i16, currentTestData.I16.data()); + engine.Put(var_i32, currentTestData.I32.data()); + engine.Put(var_i64, currentTestData.I64.data()); + engine.Put(var_u8, currentTestData.U8.data()); + engine.Put(var_u16, currentTestData.U16.data()); + engine.Put(var_u32, currentTestData.U32.data()); + engine.Put(var_u64, currentTestData.U64.data()); + engine.Put(var_r32, currentTestData.R32.data()); + engine.Put(var_r64, currentTestData.R64.data()); + engine.EndStep(); + } + + // Close the file + engine.Close(); + } + + size_t ExtraSteps = 2; + + { + // Append + adios2::IO io = adios.DeclareIO("ioAppend"); + + if (!engineName.empty()) + io.SetEngine(engineName); + else + io.SetEngine("HDF5"); + + adios2::Engine appender = io.Open(fname, adios2::Mode::Append); + + for (size_t step = NSteps; step < NSteps + ExtraSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); + + // Retrieve the variables that previously went out of scope + auto var_iString = io.InquireVariable("iString"); + EXPECT_TRUE(var_iString); + auto var_i8 = io.InquireVariable("i8"); + EXPECT_TRUE(var_i8); + auto var_i16 = io.InquireVariable("i16"); + auto var_i32 = io.InquireVariable("i32"); + auto var_i64 = io.InquireVariable("i64"); + auto var_u8 = io.InquireVariable("u8"); + auto var_u16 = io.InquireVariable("u16"); + auto var_u32 = io.InquireVariable("u32"); + auto var_u64 = io.InquireVariable("u64"); + auto var_r32 = io.InquireVariable("r32"); + auto var_r64 = io.InquireVariable("r64"); + + // Make a 1D selection to describe the local dimensions of the + // variable we write and its offsets in the global spaces + adios2::Box sel({mpiRank * Nx}, {Nx}); + + EXPECT_THROW(var_iString.SetSelection(sel), std::invalid_argument); + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + // Write each one + // fill in the variable with values from starting index to + // starting index + count + appender.BeginStep(); + appender.Put(var_iString, currentTestData.S1); + appender.Put(var_i8, currentTestData.I8.data()); + appender.Put(var_i16, currentTestData.I16.data()); + appender.Put(var_i32, currentTestData.I32.data()); + appender.Put(var_i64, currentTestData.I64.data()); + appender.Put(var_u8, currentTestData.U8.data()); + appender.Put(var_u16, currentTestData.U16.data()); + appender.Put(var_u32, currentTestData.U32.data()); + appender.Put(var_u64, currentTestData.U64.data()); + appender.Put(var_r32, currentTestData.R32.data()); + appender.Put(var_r64, currentTestData.R64.data()); + appender.EndStep(); + } + appender.Close(); + } + + { + // Read back + adios2::IO io = adios.DeclareIO("ioRead"); + if (!engineName.empty()) + io.SetEngine(engineName); + else + io.SetEngine("HDF5"); + + adios2::Engine reader = io.Open(fname, adios2::Mode::Read); + EXPECT_EQ(reader.Steps(), NSteps + ExtraSteps); + + std::string IString; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + auto var_iString = io.InquireVariable("iString"); + EXPECT_TRUE(var_iString); + EXPECT_EQ(var_iString.Steps(), NSteps + ExtraSteps); + + auto var_i8 = io.InquireVariable("i8"); + EXPECT_TRUE(var_i8); + EXPECT_EQ(var_i8.Steps(), NSteps + ExtraSteps); + + auto var_i16 = io.InquireVariable("i16"); + EXPECT_TRUE(var_i16); + EXPECT_EQ(var_i16.Steps(), NSteps + ExtraSteps); + + auto var_i32 = io.InquireVariable("i32"); + EXPECT_TRUE(var_i32); + EXPECT_EQ(var_i32.Steps(), NSteps + ExtraSteps); + + auto var_i64 = io.InquireVariable("i64"); + EXPECT_TRUE(var_i64); + EXPECT_EQ(var_i64.Steps(), NSteps + ExtraSteps); + + auto var_u8 = io.InquireVariable("u8"); + EXPECT_TRUE(var_u8); + EXPECT_EQ(var_u8.Steps(), NSteps + ExtraSteps); + + auto var_u16 = io.InquireVariable("u16"); + EXPECT_TRUE(var_u16); + EXPECT_EQ(var_u16.Steps(), NSteps + ExtraSteps); + + auto var_u32 = io.InquireVariable("u32"); + EXPECT_TRUE(var_u32); + EXPECT_EQ(var_u32.Steps(), NSteps + ExtraSteps); + + auto var_u64 = io.InquireVariable("u64"); + EXPECT_TRUE(var_u64); + EXPECT_EQ(var_u64.Steps(), NSteps + ExtraSteps); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + EXPECT_EQ(var_r32.Steps(), NSteps + ExtraSteps); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + EXPECT_EQ(var_r64.Steps(), NSteps + ExtraSteps); + + adios2::Box sel({mpiRank * Nx}, {Nx}); + + for (size_t step = 0; step < NSteps + ExtraSteps; ++step) + { + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); + + var_i8.SetStepSelection({step, 1}); + var_i8.SetSelection(sel); + reader.Get(var_i8, I8.data()); + + var_i16.SetStepSelection({step, 1}); + var_i16.SetSelection(sel); + reader.Get(var_i16, I16.data()); + + var_i32.SetStepSelection({step, 1}); + var_i32.SetSelection(sel); + reader.Get(var_i32, I32.data()); + + var_i64.SetStepSelection({step, 1}); + var_i64.SetSelection(sel); + reader.Get(var_i64, I64.data()); + + var_u8.SetStepSelection({step, 1}); + var_u8.SetSelection(sel); + reader.Get(var_u8, U8.data()); + + var_u16.SetStepSelection({step, 1}); + var_u16.SetSelection(sel); + reader.Get(var_u16, U16.data()); + + var_u32.SetStepSelection({step, 1}); + var_u32.SetSelection(sel); + reader.Get(var_u32, U32.data()); + + var_u64.SetStepSelection({step, 1}); + var_u64.SetSelection(sel); + reader.Get(var_u64, U64.data()); + + var_r32.SetStepSelection({step, 1}); + var_r32.SetSelection(sel); + reader.Get(var_r32, R32.data()); + + var_r64.SetStepSelection({step, 1}); + var_r64.SetSelection(sel); + reader.Get(var_r64, R64.data()); + + reader.Get(var_iString, IString); + reader.PerformGets(); + + EXPECT_EQ(IString, currentTestData.S1); + + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "step=" << step << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; + EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; + EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; + EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; + + EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; + EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; + EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; + EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; + + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + } + + reader.Close(); + } +} + +//****************************************************************************** +// main +//****************************************************************************** + +int main(int argc, char **argv) +{ +#if ADIOS2_USE_MPI + MPI_Init(nullptr, nullptr); +#endif + + int result; + ::testing::InitGoogleTest(&argc, argv); + + if (argc > 1) + { + engineName = std::string(argv[1]); + } + result = RUN_ALL_TESTS(); + +#if ADIOS2_USE_MPI + MPI_Finalize(); +#endif + + return result; +}