From f108c3ff1911162804bc77d83056cd73f0875bc5 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Sun, 28 Jan 2024 09:47:10 -0500 Subject: [PATCH] set params on all transports --- source/adios2/engine/bp5/BP5Reader.cpp | 2 +- .../toolkit/transportman/TransportMan.cpp | 20 ++- .../bp/operations/TestBPWriteReadBZIP2.cpp | 130 ++++++++++++++++++ 3 files changed, 147 insertions(+), 5 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 553a5901f8..cb0b9aa570 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -229,7 +229,7 @@ std::pair BP5Reader::ReadData(adios2::transportman::TransportMan { Params transportParameters; transportParameters["FailOnEOF"] = "true"; - FileManager.SetParameters(transportParameters); + FileManager.SetParameters(transportParameters, -1); } } TP endSubfile = NOW(); diff --git a/source/adios2/toolkit/transportman/TransportMan.cpp b/source/adios2/toolkit/transportman/TransportMan.cpp index aefd6820d7..f605915555 100644 --- a/source/adios2/toolkit/transportman/TransportMan.cpp +++ b/source/adios2/toolkit/transportman/TransportMan.cpp @@ -410,10 +410,22 @@ void TransportMan::ReadFile(char *buffer, const size_t size, const size_t start, void TransportMan::SetParameters(const Params ¶ms, const size_t transportIndex) { - auto itTransport = m_Transports.find(transportIndex); - CheckFile(itTransport, - ", in call to SetParameters with index " + std::to_string(transportIndex)); - itTransport->second->SetParameters(params); + if (transportIndex == -1) + { + for (auto &transportPair : m_Transports) + { + auto &transport = transportPair.second; + + transport->SetParameters(params); + } + } + else + { + auto itTransport = m_Transports.find(transportIndex); + CheckFile(itTransport, + ", in call to SetParameters with index " + std::to_string(transportIndex)); + itTransport->second->SetParameters(params); + } } void TransportMan::FlushFiles(const int transportIndex) diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp index 4e1f7f86fb..af47705a2e 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp @@ -148,6 +148,135 @@ void BZIP2Accuracy1D(const std::string accuracy) } } +void BZIP2Accuracy1DLocal(const std::string accuracy) +{ + // Each process would write a 1x8 array and all processes would + // form a mpiSize * Nx 1D array + const std::string fname("BPWR_BZIP2_1D_Local_" + accuracy + ".bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const size_t Nx = 1000; + + // Number of steps + const size_t NSteps = 1; + + std::vector r32s(Nx); + std::vector r64s(Nx); + + // range 0 to 999 + std::iota(r32s.begin(), r32s.end(), 0.f); + std::iota(r64s.begin(), r64s.end(), 0.); + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + adios2::IO io = adios.DeclareIO("TestIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + + const adios2::Dims shape{}; + const adios2::Dims start{}; + const adios2::Dims count{Nx}; + + adios2::Variable var_r32 = + io.DefineVariable("r32", shape, start, count, adios2::ConstantDims); + adios2::Variable var_r64 = + io.DefineVariable("r64", shape, start, count, adios2::ConstantDims); + + // add operations + adios2::Operator BZIP2Op = + adios.DefineOperator("BZIP2Compressor", adios2::ops::LosslessBZIP2); + + var_r32.AddOperation(BZIP2Op, {{adios2::ops::bzip2::key::blockSize100k, accuracy}}); + var_r64.AddOperation(BZIP2Op, {{adios2::ops::bzip2::key::blockSize100k, accuracy}}); + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + bpWriter.BeginStep(); + bpWriter.Put("r32", r32s.data()); + bpWriter.Put("r64", r64s.data()); + bpWriter.EndStep(); + } + + bpWriter.Close(); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + + unsigned int t = 0; + std::vector decompressedR32s; + std::vector decompressedR64s; + + while (bpReader.BeginStep() == adios2::StepStatus::OK) + { + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::LocalArray); + ASSERT_EQ(var_r32.Steps(), NSteps); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::LocalArray); + ASSERT_EQ(var_r64.Steps(), NSteps); + auto r32_info = bpReader.BlocksInfo(var_r32, -1); + auto r64_info = bpReader.BlocksInfo(var_r64, -1); + + var_r32.SetBlockSelection(mpiRank); + var_r64.SetBlockSelection(mpiRank); + + bpReader.Get(var_r32, decompressedR32s); + bpReader.Get(var_r64, decompressedR64s); + bpReader.EndStep(); + + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + ASSERT_EQ(decompressedR32s[i], r32s[i]) << msg; + ASSERT_EQ(decompressedR64s[i], r64s[i]) << msg; + } + ++t; + } + + EXPECT_EQ(t, NSteps); + + bpReader.Close(); + } +} + void BZIP2Accuracy2D(const std::string accuracy) { // Each process would write a 1x8 array and all processes would @@ -830,6 +959,7 @@ class BPWriteReadBZIP2 : public ::testing::TestWithParam }; TEST_P(BPWriteReadBZIP2, ADIOS2BPWriteReadBZIP21D) { BZIP2Accuracy1D(GetParam()); } +TEST_P(BPWriteReadBZIP2, ADIOS2BPWriteReadBZIP21DLocal) { BZIP2Accuracy1DLocal(GetParam()); } TEST_P(BPWriteReadBZIP2, ADIOS2BPWriteReadBZIP22D) { BZIP2Accuracy2D(GetParam()); } TEST_P(BPWriteReadBZIP2, ADIOS2BPWriteReadBZIP23D) { BZIP2Accuracy3D(GetParam()); } TEST_P(BPWriteReadBZIP2, ADIOS2BPWriteReadBZIP21DSel) { BZIP2Accuracy1DSel(GetParam()); }