diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 6c8d01b5bf..1ebac14e55 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -171,9 +171,11 @@ if(ADIOS2_HAVE_MPI) engine/ssc/SscWriter.cpp engine/ssc/SscWriterBase.cpp engine/ssc/SscWriterGeneric.cpp + engine/ssc/SscWriterNaive.cpp engine/ssc/SscReader.cpp engine/ssc/SscReaderBase.cpp engine/ssc/SscReaderGeneric.cpp + engine/ssc/SscReaderNaive.cpp ) set_property(TARGET adios2_core_mpi PROPERTY EXPORT_NAME core_mpi) set_property(TARGET adios2_core_mpi PROPERTY OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core_mpi) diff --git a/source/adios2/engine/ssc/SscHelper.cpp b/source/adios2/engine/ssc/SscHelper.cpp index 24aec23675..8fc70fb4f6 100644 --- a/source/adios2/engine/ssc/SscHelper.cpp +++ b/source/adios2/engine/ssc/SscHelper.cpp @@ -243,55 +243,36 @@ void SerializeAttributes(IO &input, Buffer &output) } } -void Deserialize(const Buffer &input, BlockVecVec &output, IO &io, - const bool regVars, const bool regAttrs) +void DeserializeAttribute(const Buffer &input, uint64_t &pos, IO &io, + const bool regIO) { - for (auto &i : output) - { - i.clear(); - } + const DataType type = static_cast(input[pos]); + ++pos; - uint64_t pos = 2; + uint8_t nameSize = input[pos]; + ++pos; - uint64_t blockSize = input.value(pos); + std::vector namev(nameSize); + std::memcpy(namev.data(), input.data(pos), nameSize); + std::string name = std::string(namev.begin(), namev.end()); + pos += nameSize; + uint64_t size = input.value(pos); pos += 8; - while (pos < blockSize) + if (regIO) { - - uint8_t shapeId = input[pos]; - ++pos; - - if (shapeId == 66) + const auto &attributes = io.GetAttributes(); + auto it = attributes.find(name); + if (it == attributes.end()) { - const DataType type = static_cast(input[pos]); - ++pos; - - uint8_t nameSize = input[pos]; - ++pos; - - std::vector namev(nameSize); - std::memcpy(namev.data(), input.data(pos), nameSize); - std::string name = std::string(namev.begin(), namev.end()); - pos += nameSize; - - uint64_t size = input.value(pos); - pos += 8; - - if (regAttrs) + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (type == DataType::String) { - const auto &attributes = io.GetAttributes(); - auto it = attributes.find(name); - if (it == attributes.end()) - { - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (type == DataType::String) - { - io.DefineAttribute( - name, std::string(input.data(pos), size)); - } + io.DefineAttribute( + name, std::string(input.data(pos), size)); + } #define declare_type(T) \ else if (type == helper::GetDataType()) \ { \ @@ -304,75 +285,73 @@ void Deserialize(const Buffer &input, BlockVecVec &output, IO &io, io.DefineAttribute(name, input.data(pos), size / sizeof(T)); \ } \ } - ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_type) + ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_type) #undef declare_type - else - { - helper::Throw( - "Engine", "SscHelper", "Deserialize", - "unknown attribute data type"); - } - } + else + { + helper::Throw( + "Engine", "SscHelper", "Deserialize", + "unknown attribute data type"); } - pos += size; } - else - { - int rank = input.value(pos); - pos += 4; - output[rank].emplace_back(); - auto &b = output[rank].back(); - b.shapeId = static_cast(shapeId); + } + pos += size; +} - uint8_t nameSize = input[pos]; - ++pos; +void DeserializeVariable(const Buffer &input, const ShapeID shapeId, + uint64_t &pos, BlockInfo &b, IO &io, const bool regIO) +{ + b.shapeId = static_cast(shapeId); - std::vector name(nameSize); - std::memcpy(name.data(), input.data(pos), nameSize); - b.name = std::string(name.begin(), name.end()); - pos += nameSize; + uint8_t nameSize = input[pos]; + ++pos; - b.type = static_cast(input[pos]); - ++pos; + std::vector name(nameSize); + std::memcpy(name.data(), input.data(pos), nameSize); + b.name = std::string(name.begin(), name.end()); + pos += nameSize; - uint8_t shapeSize = input[pos]; - ++pos; - b.shape.resize(shapeSize); - b.start.resize(shapeSize); - b.count.resize(shapeSize); + b.type = static_cast(input[pos]); + ++pos; - std::memcpy(b.shape.data(), input.data(pos), 8 * shapeSize); - pos += 8 * shapeSize; + uint8_t shapeSize = input[pos]; + ++pos; + b.shape.resize(shapeSize); + b.start.resize(shapeSize); + b.count.resize(shapeSize); - std::memcpy(b.start.data(), input.data(pos), 8 * shapeSize); - pos += 8 * shapeSize; + std::memcpy(b.shape.data(), input.data(pos), 8 * shapeSize); + pos += 8 * shapeSize; - std::memcpy(b.count.data(), input.data(pos), 8 * shapeSize); - pos += 8 * shapeSize; + std::memcpy(b.start.data(), input.data(pos), 8 * shapeSize); + pos += 8 * shapeSize; - b.bufferStart = input.value(pos); - pos += 8; + std::memcpy(b.count.data(), input.data(pos), 8 * shapeSize); + pos += 8 * shapeSize; - b.bufferCount = input.value(pos); - pos += 8; + b.bufferStart = input.value(pos); + pos += 8; - uint8_t valueSize = input[pos]; - pos++; - b.value.resize(valueSize); - if (valueSize > 0) - { - std::memcpy(b.value.data(), input.data() + pos, valueSize); - pos += valueSize; - } + b.bufferCount = input.value(pos); + pos += 8; - if (regVars) - { - if (b.type == DataType::None) - { - helper::Throw( - "Engine", "SscHelper", "Deserialize", - "unknown variable data type"); - } + uint8_t valueSize = input[pos]; + pos++; + b.value.resize(valueSize); + if (valueSize > 0) + { + std::memcpy(b.value.data(), input.data() + pos, valueSize); + pos += valueSize; + } + + if (regIO) + { + if (b.type == DataType::None) + { + helper::Throw("Engine", "SscHelper", + "Deserialize", + "unknown variable data type"); + } #define declare_type(T) \ else if (b.type == helper::GetDataType()) \ { \ @@ -404,15 +383,50 @@ void Deserialize(const Buffer &input, BlockVecVec &output, IO &io, } \ } \ } - ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type - else - { - helper::Throw( - "Engine", "SscHelper", "Deserialize", - "unknown variable data type"); - } - } + else + { + helper::Throw("Engine", "SscHelper", + "Deserialize", + "unknown variable data type"); + } + } +} + +void Deserialize(const Buffer &input, BlockVecVec &output, IO &io, + const bool regVars, const bool regAttrs) +{ + for (auto &i : output) + { + i.clear(); + } + + uint64_t pos = 2; + + uint64_t blockSize = input.value(pos); + + pos += 8; + + while (pos < blockSize) + { + + uint8_t shapeId = input[pos]; + ++pos; + + if (shapeId == 66) + { + DeserializeAttribute(input, pos, io, regAttrs); + } + else + { + int rank = input.value(pos); + pos += 4; + output[rank].emplace_back(); + auto &b = output[rank].back(); + + DeserializeVariable(input, static_cast(shapeId), pos, b, + io, regVars); } } } diff --git a/source/adios2/engine/ssc/SscHelper.h b/source/adios2/engine/ssc/SscHelper.h index 7c76f960dc..c0c953f1ba 100644 --- a/source/adios2/engine/ssc/SscHelper.h +++ b/source/adios2/engine/ssc/SscHelper.h @@ -148,6 +148,10 @@ RankPosMap CalculateOverlap(BlockVecVec &globalPattern, void SerializeVariables(const BlockVec &input, Buffer &output, const int rank); void SerializeAttributes(IO &input, Buffer &output); +void DeserializeVariable(const Buffer &input, const ShapeID shapeId, + uint64_t &pos, BlockInfo &b, IO &io, const bool regIO); +void DeserializeAttribute(const Buffer &input, uint64_t &pos, IO &io, + const bool regIO); void Deserialize(const Buffer &input, BlockVecVec &output, IO &io, const bool regVars, const bool regAttrs); void AggregateMetadata(const Buffer &localBuffer, Buffer &globalBuffer, diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index da51c63c6d..faafb9439f 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -10,6 +10,7 @@ #include "SscReader.h" #include "SscReaderGeneric.h" +#include "SscReaderNaive.h" #include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosString.h" #include @@ -37,6 +38,8 @@ SscReader::SscReader(IO &io, const std::string &name, const Mode mode, } else if (m_EngineMode == "naive") { + m_EngineInstance = std::make_shared( + io, name, mode, CommAsMPI(m_Comm)); } } @@ -44,12 +47,14 @@ StepStatus SscReader::BeginStep(StepMode stepMode, const float timeoutSeconds) { PERFSTUBS_SCOPED_TIMER_FUNC(); - helper::Log("Engine", "SSCReader", "BeginStep", + auto ret = m_EngineInstance->BeginStep(stepMode, timeoutSeconds, + m_ReaderSelectionsLocked); + + helper::Log("Engine", "SscReader", "BeginStep", std::to_string(CurrentStep()), 0, m_Comm.Rank(), 5, m_Verbosity, helper::LogMode::INFO); - return m_EngineInstance->BeginStep(stepMode, timeoutSeconds, - m_ReaderSelectionsLocked); + return ret; } size_t SscReader::CurrentStep() const diff --git a/source/adios2/engine/ssc/SscReaderGeneric.cpp b/source/adios2/engine/ssc/SscReaderGeneric.cpp index 6fdad618d6..24c6633ede 100644 --- a/source/adios2/engine/ssc/SscReaderGeneric.cpp +++ b/source/adios2/engine/ssc/SscReaderGeneric.cpp @@ -9,7 +9,6 @@ */ #include "SscReaderGeneric.tcc" -#include "adios2/helper/adiosMemory.h" namespace adios2 { diff --git a/source/adios2/engine/ssc/SscReaderGeneric.tcc b/source/adios2/engine/ssc/SscReaderGeneric.tcc index 01478672f3..4f1a523677 100644 --- a/source/adios2/engine/ssc/SscReaderGeneric.tcc +++ b/source/adios2/engine/ssc/SscReaderGeneric.tcc @@ -8,6 +8,9 @@ * Author: Jason Wang */ +#ifndef ADIOS2_ENGINE_SSCREADERGENERIC_TCC_ +#define ADIOS2_ENGINE_SSCREADERGENERIC_TCC_ + #include "SscReaderGeneric.h" #include "adios2/helper/adiosMemory.h" @@ -212,3 +215,5 @@ SscReaderGeneric::BlocksInfoCommon(const Variable &variable, } } } + +#endif // ADIOS2_ENGINE_SSCREADERGENERIC_TCC_ diff --git a/source/adios2/engine/ssc/SscReaderNaive.cpp b/source/adios2/engine/ssc/SscReaderNaive.cpp new file mode 100644 index 0000000000..302eddb449 --- /dev/null +++ b/source/adios2/engine/ssc/SscReaderNaive.cpp @@ -0,0 +1,117 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscReaderNaive.cpp + * + * Created on: Mar 7, 2022 + * Author: Jason Wang + */ + +#include "SscReaderNaive.tcc" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +SscReaderNaive::SscReaderNaive(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm) +: SscReaderBase(io, name, mode, comm) +{ +} + +StepStatus SscReaderNaive::BeginStep(const StepMode stepMode, + const float timeoutSeconds, + const bool readerLocked) +{ + + m_Buffer.clear(); + m_BlockMap.clear(); + + ++m_CurrentStep; + + size_t globalSize; + + if (m_ReaderRank == 0) + { + MPI_Recv(&globalSize, 1, MPI_UNSIGNED_LONG_LONG, + m_WriterMasterStreamRank, 0, m_StreamComm, MPI_STATUS_IGNORE); + m_Buffer.resize(globalSize); + MPI_Recv(m_Buffer.data(), globalSize, MPI_UNSIGNED_LONG_LONG, + m_WriterMasterStreamRank, 0, m_StreamComm, MPI_STATUS_IGNORE); + } + + MPI_Bcast(&globalSize, 1, MPI_UNSIGNED_LONG_LONG, 0, m_ReaderComm); + if (m_ReaderRank != 0) + { + m_Buffer.resize(globalSize); + } + if (globalSize == 1) + { + return StepStatus::EndOfStream; + } + MPI_Bcast(m_Buffer.data(), globalSize, MPI_CHAR, 0, m_ReaderComm); + + uint64_t pos = 0; + while (pos < m_Buffer.size()) + { + uint64_t start = pos; + uint64_t end = pos + m_Buffer.value(pos); + pos += m_Buffer.value(pos + 8); + + while (pos < end) + { + uint8_t shapeId = m_Buffer[pos]; + ++pos; + + if (shapeId == 66) + { + DeserializeAttribute(m_Buffer, pos, m_IO, true); + } + else + { + pos += 4; + ssc::BlockInfo b; + DeserializeVariable(m_Buffer, static_cast(shapeId), + pos, b, m_IO, true); + b.bufferStart += start; + m_BlockMap[b.name].push_back(b); + } + } + } + + return StepStatus::OK; +} + +size_t SscReaderNaive::CurrentStep() { return m_CurrentStep; } + +void SscReaderNaive::EndStep(const bool readerLocked) {} + +void SscReaderNaive::PerformGets() {} + +void SscReaderNaive::Close(const int transportIndex) {} + +#define declare_type(T) \ + void SscReaderNaive::GetDeferred(Variable &variable, T *data) \ + { \ + helper::Log("Engine", "SSCReader", "GetDeferred", variable.m_Name, 0, \ + m_ReaderRank, 5, m_Verbosity, helper::LogMode::INFO); \ + GetDeferredCommon(variable, data); \ + } \ + std::vector::BPInfo> SscReaderNaive::BlocksInfo( \ + const Variable &variable, const size_t step) const \ + { \ + return BlocksInfoCommon(variable, step); \ + } +ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +} +} +} +} diff --git a/source/adios2/engine/ssc/SscReaderNaive.h b/source/adios2/engine/ssc/SscReaderNaive.h new file mode 100644 index 0000000000..636d327351 --- /dev/null +++ b/source/adios2/engine/ssc/SscReaderNaive.h @@ -0,0 +1,65 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscReaderNaive.h + * + * Created on: Mar 7, 2022 + * Author: Jason Wang + */ + +#ifndef ADIOS2_ENGINE_SSCREADERNAIVE_H_ +#define ADIOS2_ENGINE_SSCREADERNAIVE_H_ + +#include "SscReaderBase.h" +#include "adios2/core/IO.h" +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +class SscReaderNaive : public SscReaderBase +{ + +public: + SscReaderNaive(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm); + ~SscReaderNaive() = default; + + StepStatus BeginStep(const StepMode mode, const float timeoutSeconds, + const bool readerLocked) final; + size_t CurrentStep() final; + void PerformGets() final; + void EndStep(const bool readerLocked) final; + void Close(const int transportIndex) final; + +#define declare_type(T) \ + void GetDeferred(Variable &, T *) final; \ + std::vector::BPInfo> BlocksInfo( \ + const Variable &variable, const size_t step) const final; + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +private: + template + std::vector::BPInfo> + BlocksInfoCommon(const Variable &variable, const size_t step) const; + + template + void GetDeferredCommon(Variable &variable, T *data); + + std::unordered_map m_BlockMap; +}; + +} +} +} +} + +#endif // ADIOS2_ENGINE_SSCREADERNAIVE_H_ diff --git a/source/adios2/engine/ssc/SscReaderNaive.tcc b/source/adios2/engine/ssc/SscReaderNaive.tcc new file mode 100644 index 0000000000..52d45c3c42 --- /dev/null +++ b/source/adios2/engine/ssc/SscReaderNaive.tcc @@ -0,0 +1,95 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscReaderNaive.tcc + * + * Created on: Mar 7, 2022 + * Author: Jason Wang + */ + +#ifndef ADIOS2_ENGINE_SSCREADERNAIVE_TCC_ +#define ADIOS2_ENGINE_SSCREADERNAIVE_TCC_ + +#include "SscReaderNaive.h" +#include "adios2/helper/adiosMemory.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +template <> +void SscReaderNaive::GetDeferredCommon(Variable &variable, + std::string *data) +{ + variable.SetData(data); + + for (const auto &b : m_BlockMap[variable.m_Name]) + { + if (b.name == variable.m_Name) + { + *data = + std::string(m_Buffer.data() + b.bufferStart, + m_Buffer.data() + b.bufferStart + b.bufferCount); + variable.m_Value = *data; + variable.m_Min = *data; + variable.m_Max = *data; + } + } +} + +template +void SscReaderNaive::GetDeferredCommon(Variable &variable, T *data) +{ + variable.SetData(data); + + Dims vStart = variable.m_Start; + Dims vCount = variable.m_Count; + Dims vShape = variable.m_Shape; + + if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) + { + std::reverse(vStart.begin(), vStart.end()); + std::reverse(vCount.begin(), vCount.end()); + std::reverse(vShape.begin(), vShape.end()); + } + + for (const auto &b : m_BlockMap[variable.m_Name]) + { + if (b.shapeId == ShapeID::GlobalArray || + b.shapeId == ShapeID::LocalArray) + { + helper::NdCopy(m_Buffer.data() + b.bufferStart, b.start, + b.count, true, true, reinterpret_cast(data), + vStart, vCount, true, true, sizeof(T)); + } + else if (b.shapeId == ShapeID::GlobalValue || + b.shapeId == ShapeID::LocalValue) + { + std::memcpy(data, m_Buffer.data() + b.bufferStart, b.bufferCount); + } + } +} + +template +std::vector::BPInfo> +SscReaderNaive::BlocksInfoCommon(const Variable &variable, + const size_t step) const +{ + + std::vector::BPInfo> ret; + + return ret; +} + +} +} +} +} + +#endif // ADIOS2_ENGINE_SSCREADERNAIVE_TCC_ diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp index ae342a04ba..6a9fd31f50 100644 --- a/source/adios2/engine/ssc/SscWriter.cpp +++ b/source/adios2/engine/ssc/SscWriter.cpp @@ -10,6 +10,7 @@ #include "SscWriter.h" #include "SscWriterGeneric.h" +#include "SscWriterNaive.h" #include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosString.h" #include @@ -37,6 +38,8 @@ SscWriter::SscWriter(IO &io, const std::string &name, const Mode mode, } else if (m_EngineMode == "naive") { + m_EngineInstance = std::make_shared( + io, name, mode, CommAsMPI(m_Comm)); } } @@ -44,12 +47,14 @@ StepStatus SscWriter::BeginStep(StepMode mode, const float timeoutSeconds) { PERFSTUBS_SCOPED_TIMER_FUNC(); + auto ret = m_EngineInstance->BeginStep(mode, timeoutSeconds, + m_WriterDefinitionsLocked); + helper::Log("Engine", "SSCWriter", "BeginStep", std::to_string(CurrentStep()), 0, m_Comm.Rank(), 5, m_Verbosity, helper::LogMode::INFO); - return m_EngineInstance->BeginStep(mode, timeoutSeconds, - m_WriterDefinitionsLocked); + return ret; } size_t SscWriter::CurrentStep() const diff --git a/source/adios2/engine/ssc/SscWriterGeneric.h b/source/adios2/engine/ssc/SscWriterGeneric.h index 48d30a567a..24a63d58d7 100644 --- a/source/adios2/engine/ssc/SscWriterGeneric.h +++ b/source/adios2/engine/ssc/SscWriterGeneric.h @@ -2,7 +2,7 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * SscWriterBase.h + * SscWriterGeneric.h * * Created on: Mar 3, 2022 * Author: Jason Wang diff --git a/source/adios2/engine/ssc/SscWriterGeneric.tcc b/source/adios2/engine/ssc/SscWriterGeneric.tcc index ba25f67335..5f40a581ba 100644 --- a/source/adios2/engine/ssc/SscWriterGeneric.tcc +++ b/source/adios2/engine/ssc/SscWriterGeneric.tcc @@ -8,6 +8,9 @@ * Author: Jason Wang */ +#ifndef ADIOS2_ENGINE_SSCWRITERGENERIC_TCC_ +#define ADIOS2_ENGINE_SSCWRITERGENERIC_TCC_ + #include "SscWriterGeneric.h" namespace adios2 @@ -145,3 +148,5 @@ void SscWriterGeneric::PutDeferredCommon(Variable &variable, const T *data) } } } + +#endif // ADIOS2_ENGINE_SSCWRITERGENERIC_TCC_ diff --git a/source/adios2/engine/ssc/SscWriterNaive.cpp b/source/adios2/engine/ssc/SscWriterNaive.cpp new file mode 100644 index 0000000000..604ea5f16f --- /dev/null +++ b/source/adios2/engine/ssc/SscWriterNaive.cpp @@ -0,0 +1,108 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscWriterNaive.cpp + * + * Created on: Mar 7, 2022 + * Author: Jason Wang + */ + +#include "SscWriterNaive.tcc" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +SscWriterNaive::SscWriterNaive(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm) +: SscWriterBase(io, name, mode, comm) +{ +} + +StepStatus SscWriterNaive::BeginStep(const StepMode mode, + const float timeoutSeconds, + const bool writerLocked) +{ + ++m_CurrentStep; + + m_Buffer.clear(); + m_Buffer.resize(16); + m_Metadata.clear(); + + return StepStatus::OK; +} + +size_t SscWriterNaive::CurrentStep() { return m_CurrentStep; } + +void SscWriterNaive::PerformPuts() {} + +void SscWriterNaive::EndStep(const bool writerLocked) +{ + m_Buffer.value() = m_Buffer.size(); + m_Buffer.value(8) = m_Buffer.size(); + + ssc::SerializeVariables(m_Metadata, m_Buffer, m_WriterRank); + + if (m_WriterRank == 0) + { + ssc::SerializeAttributes(m_IO, m_Buffer); + } + + int localSize = static_cast(m_Buffer.value()); + std::vector localSizes(m_WriterSize); + MPI_Gather(&localSize, 1, MPI_INT, localSizes.data(), 1, MPI_INT, 0, + m_WriterComm); + size_t globalSize = + std::accumulate(localSizes.begin(), localSizes.end(), 0); + ssc::Buffer globalBuffer(globalSize); + + std::vector displs(m_WriterSize); + for (size_t i = 1; i < static_cast(m_WriterSize); ++i) + { + displs[i] = displs[i - 1] + localSizes[i - 1]; + } + + MPI_Gatherv(m_Buffer.data(), localSize, MPI_CHAR, globalBuffer.data(), + localSizes.data(), displs.data(), MPI_CHAR, 0, m_WriterComm); + + if (m_WriterRank == 0) + { + MPI_Send(&globalSize, 1, MPI_UNSIGNED_LONG_LONG, + m_ReaderMasterStreamRank, 0, m_StreamComm); + MPI_Send(globalBuffer.data(), globalSize, MPI_CHAR, + m_ReaderMasterStreamRank, 0, m_StreamComm); + } +} + +void SscWriterNaive::Close(const int transportIndex) +{ + + uint64_t globalSize = 1; + ssc::Buffer globalBuffer(globalSize); + if (m_WriterRank == 0) + { + MPI_Send(&globalSize, 1, MPI_UNSIGNED_LONG_LONG, + m_ReaderMasterStreamRank, 0, m_StreamComm); + MPI_Send(globalBuffer.data(), globalSize, MPI_CHAR, + m_ReaderMasterStreamRank, 0, m_StreamComm); + } +} + +#define declare_type(T) \ + void SscWriterNaive::PutDeferred(Variable &variable, const T *data) \ + { \ + PutDeferredCommon(variable, data); \ + } +ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +} +} +} +} diff --git a/source/adios2/engine/ssc/SscWriterNaive.h b/source/adios2/engine/ssc/SscWriterNaive.h new file mode 100644 index 0000000000..0e6b9c6f71 --- /dev/null +++ b/source/adios2/engine/ssc/SscWriterNaive.h @@ -0,0 +1,58 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscWriterNaive.h + * + * Created on: Mar 7, 2022 + * Author: Jason Wang + */ + +#ifndef ADIOS2_ENGINE_SSCWRITERNAIVE_H_ +#define ADIOS2_ENGINE_SSCWRITERNAIVE_H_ + +#include "SscWriterBase.h" +#include "adios2/core/IO.h" +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +class SscWriterNaive : public SscWriterBase +{ + +public: + SscWriterNaive(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm); + ~SscWriterNaive() = default; + + StepStatus BeginStep(const StepMode mode, const float timeoutSeconds, + const bool writerLocked) final; + size_t CurrentStep() final; + void PerformPuts() final; + void EndStep(const bool writerLocked) final; + void Close(const int transportIndex) final; + +#define declare_type(T) void PutDeferred(Variable &, const T *) final; + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +private: + template + void PutDeferredCommon(Variable &variable, const T *values); + + ssc::BlockVec m_Metadata; +}; + +} +} +} +} + +#endif // ADIOS2_ENGINE_SSCWRITENAIVE_H_ diff --git a/source/adios2/engine/ssc/SscWriterNaive.tcc b/source/adios2/engine/ssc/SscWriterNaive.tcc new file mode 100644 index 0000000000..c087a04a0d --- /dev/null +++ b/source/adios2/engine/ssc/SscWriterNaive.tcc @@ -0,0 +1,110 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscWriterNaive.tcc + * + * Created on: Mar 7, 2022 + * Author: Jason Wang + */ + +#ifndef ADIOS2_ENGINE_SSCWRITERNAIVE_TCC_ +#define ADIOS2_ENGINE_SSCWRITERNAIVE_TCC_ + +#include "SscWriterNaive.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +template <> +void SscWriterNaive::PutDeferredCommon(Variable &variable, + const std::string *data) +{ + variable.SetData(data); + m_Metadata.emplace_back(); + auto &b = m_Metadata.back(); + b.name = variable.m_Name; + b.type = DataType::String; + b.shapeId = variable.m_ShapeID; + b.shape = variable.m_Shape; + b.start = variable.m_Start; + b.count = variable.m_Count; + b.bufferStart = m_Buffer.size(); + b.bufferCount = data->size(); + m_Buffer.resize(b.bufferStart + b.bufferCount); + std::memcpy(m_Buffer.data() + b.bufferStart, data->data(), data->size()); + b.value.resize(data->size()); + std::memcpy(b.value.data(), data->data(), data->size()); +} + +template +void SscWriterNaive::PutDeferredCommon(Variable &variable, const T *data) +{ + + if ((variable.m_ShapeID == ShapeID::GlobalValue || + variable.m_ShapeID == ShapeID::LocalValue || + variable.m_Type == DataType::String) && + m_WriterRank != 0) + { + return; + } + + variable.SetData(data); + + Dims vStart = variable.m_Start; + Dims vCount = variable.m_Count; + Dims vShape = variable.m_Shape; + + if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) + { + std::reverse(vStart.begin(), vStart.end()); + std::reverse(vCount.begin(), vCount.end()); + std::reverse(vShape.begin(), vShape.end()); + } + + bool found = false; + for (const auto &b : m_Metadata) + { + if (b.name == variable.m_Name && ssc::AreSameDims(vStart, b.start) && + ssc::AreSameDims(vCount, b.count) && + ssc::AreSameDims(vShape, b.shape)) + { + std::memcpy(m_Buffer.data() + b.bufferStart, data, b.bufferCount); + found = true; + } + } + + if (!found) + { + m_Metadata.emplace_back(); + auto &b = m_Metadata.back(); + b.name = variable.m_Name; + b.type = helper::GetDataType(); + b.shapeId = variable.m_ShapeID; + b.shape = vShape; + b.start = vStart; + b.count = vCount; + b.bufferStart = m_Buffer.size(); + b.bufferCount = ssc::TotalDataSize(b.count, b.type, b.shapeId); + m_Buffer.resize(b.bufferStart + b.bufferCount); + std::memcpy(m_Buffer.data() + b.bufferStart, data, b.bufferCount); + if (b.shapeId == ShapeID::GlobalValue || + b.shapeId == ShapeID::LocalValue) + { + b.value.resize(sizeof(*data)); + std::memcpy(b.value.data(), data, b.bufferCount); + } + } +} +} +} +} +} + +#endif // ADIOS2_ENGINE_SSCWRITERNAIVE_TCC_ diff --git a/testing/adios2/engine/ssc/TestSsc7d.cpp b/testing/adios2/engine/ssc/TestSsc7d.cpp index 730979421e..6238e7f1f7 100644 --- a/testing/adios2/engine/ssc/TestSsc7d.cpp +++ b/testing/adios2/engine/ssc/TestSsc7d.cpp @@ -192,36 +192,72 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSsc7d) { - std::string filename = "TestSsc7d"; - adios2::Params engineParams = {}; + { + std::string filename = "TestSsc7d"; + adios2::Params engineParams = {}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); - Dims shape = {10, 2, 2, (size_t)mpiSize, 2, 8, 10}; - Dims start = {0, 0, 0, (size_t)mpiRank, 0, 0, 0}; - Dims count = {10, 2, 2, 1, 2, 8, 10}; - size_t steps = 20; + Dims shape = {10, 2, 2, (size_t)mpiSize, 2, 8, 10}; + Dims start = {0, 0, 0, (size_t)mpiRank, 0, 0, 0}; + Dims count = {10, 2, 2, 1, 2, 8, 10}; + size_t steps = 20; - if (mpiGroup == 0) - { - Writer(shape, start, count, steps, engineParams, filename); - } + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (mpiGroup == 1) - { - Reader(shape, start, count, steps, engineParams, filename); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); + { + std::string filename = "TestSsc7dNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + Dims shape = {10, 2, 2, (size_t)mpiSize, 2, 8, 10}; + Dims start = {0, 0, 0, (size_t)mpiRank, 0, 0, 0}; + Dims count = {10, 2, 2, 1, 2, 8, 10}; + size_t steps = 20; + + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); + } } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscBase.cpp b/testing/adios2/engine/ssc/TestSscBase.cpp index 0409ff9a61..6ce18c2cc5 100644 --- a/testing/adios2/engine/ssc/TestSscBase.cpp +++ b/testing/adios2/engine/ssc/TestSscBase.cpp @@ -242,36 +242,57 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscBase) { - std::string filename = "TestSscBase"; - adios2::Params engineParams = {{"Verbose", "0"}}; - - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); - - Dims shape = {10, (size_t)mpiSize * 2}; - Dims start = {2, (size_t)mpiRank * 2}; - Dims count = {5, 2}; - size_t steps = 100; - - if (mpiGroup == 0) { - Writer(shape, start, count, steps, engineParams, filename); + std::string filename = "TestSscBase"; + adios2::Params engineParams = {{"Verbose", "0"}}; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 100; + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + MPI_Barrier(MPI_COMM_WORLD); } - - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - - if (mpiGroup == 1) { - Reader(shape, start, count, steps, engineParams, filename); + std::string filename = "TestSscBaseNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 100; + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + MPI_Barrier(MPI_COMM_WORLD); } - - MPI_Barrier(MPI_COMM_WORLD); } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscSingleStep.cpp b/testing/adios2/engine/ssc/TestSscSingleStep.cpp index bf8b673788..6c4fe042a2 100644 --- a/testing/adios2/engine/ssc/TestSscSingleStep.cpp +++ b/testing/adios2/engine/ssc/TestSscSingleStep.cpp @@ -243,36 +243,72 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscSingleStep) { - std::string filename = "TestSscSingleStep"; - adios2::Params engineParams = {{"Verbose", "0"}}; + { + std::string filename = "TestSscSingleStep"; + adios2::Params engineParams = {{"Verbose", "0"}}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); - Dims shape = {10, (size_t)mpiSize * 2}; - Dims start = {2, (size_t)mpiRank * 2}; - Dims count = {5, 2}; - size_t steps = 1; + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 1; - if (mpiGroup == 0) - { - Writer(shape, start, count, steps, engineParams, filename); - } + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (mpiGroup == 1) - { - Reader(shape, start, count, steps, engineParams, filename); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); + { + std::string filename = "TestSscSingleStepNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 1; + + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); + } } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscSuperLarge.cpp b/testing/adios2/engine/ssc/TestSscSuperLarge.cpp index f485658c42..4bd0b60606 100644 --- a/testing/adios2/engine/ssc/TestSscSuperLarge.cpp +++ b/testing/adios2/engine/ssc/TestSscSuperLarge.cpp @@ -242,36 +242,72 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscSuperLarge) { - std::string filename = "TestSscSuperLarge"; - adios2::Params engineParams = {{"Verbose", "0"}}; + { + std::string filename = "TestSscSuperLarge"; + adios2::Params engineParams = {{"Verbose", "0"}}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); - Dims shape = {100, (size_t)mpiSize * 10}; - Dims start = {10, (size_t)mpiRank * 10}; - Dims count = {90, 10}; - size_t steps = 50; + Dims shape = {100, (size_t)mpiSize * 10}; + Dims start = {10, (size_t)mpiRank * 10}; + Dims count = {90, 10}; + size_t steps = 50; - if (mpiGroup == 0) - { - Writer(shape, start, count, steps, engineParams, filename); - } + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (mpiGroup == 1) - { - Reader(shape, start, count, steps, engineParams, filename); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); + { + std::string filename = "TestSscSuperLargeNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + Dims shape = {100, (size_t)mpiSize * 10}; + Dims start = {10, (size_t)mpiRank * 10}; + Dims count = {90, 10}; + size_t steps = 50; + + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); + } } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscUnbalanced.cpp b/testing/adios2/engine/ssc/TestSscUnbalanced.cpp index 17be4181ac..39aab4b19b 100644 --- a/testing/adios2/engine/ssc/TestSscUnbalanced.cpp +++ b/testing/adios2/engine/ssc/TestSscUnbalanced.cpp @@ -215,38 +215,76 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscUnbalanced) { - std::string filename = "TestSscUnbalanced"; - adios2::Params engineParams = {{"Verbose", "0"}}; + { + std::string filename = "TestSscUnbalanced"; + adios2::Params engineParams = {{"Verbose", "0"}}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); - Dims shape = {10, (size_t)mpiSize * 2}; - Dims start = {2, (size_t)mpiRank * 2}; - Dims count = {5, 2}; - size_t steps = 100; + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 100; - if (mpiGroup == 0) - { - Writer(shape, start, count, steps, engineParams, filename); - } + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (mpiGroup == 1) - { - start = {2, 0}; - count = {5, 2}; - Reader(shape, start, count, steps, engineParams, filename); + if (mpiGroup == 1) + { + start = {2, 0}; + count = {5, 2}; + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); + { + std::string filename = "TestSscUnbalancedNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 100; + + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + start = {2, 0}; + count = {5, 2}; + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); + } } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscVaryingSteps.cpp b/testing/adios2/engine/ssc/TestSscVaryingSteps.cpp index f80b1c121f..639ca67522 100644 --- a/testing/adios2/engine/ssc/TestSscVaryingSteps.cpp +++ b/testing/adios2/engine/ssc/TestSscVaryingSteps.cpp @@ -288,36 +288,72 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscVaryingSteps) { - std::string filename = "TestSscVaryingSteps"; - adios2::Params engineParams = {}; + { + std::string filename = "TestSscVaryingSteps"; + adios2::Params engineParams = {}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); - Dims shape = {1, (size_t)mpiSize * 2}; - Dims start = {0, (size_t)mpiRank * 2}; - Dims count = {1, 2}; - size_t steps = 100; + Dims shape = {1, (size_t)mpiSize * 2}; + Dims start = {0, (size_t)mpiRank * 2}; + Dims count = {1, 2}; + size_t steps = 100; - if (mpiGroup == 0) - { - Writer(shape, start, count, steps, engineParams, filename); - } + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (mpiGroup == 1) - { - Reader(shape, start, count, steps, engineParams, filename); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); + { + std::string filename = "TestSscVaryingStepsNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + Dims shape = {1, (size_t)mpiSize * 2}; + Dims start = {0, (size_t)mpiRank * 2}; + Dims count = {1, 2}; + size_t steps = 100; + + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); + } } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscWriterMultiblock.cpp b/testing/adios2/engine/ssc/TestSscWriterMultiblock.cpp index 02fa6cdfe6..b0d376d19f 100644 --- a/testing/adios2/engine/ssc/TestSscWriterMultiblock.cpp +++ b/testing/adios2/engine/ssc/TestSscWriterMultiblock.cpp @@ -255,39 +255,78 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscWriterMultiblock) { - std::string filename = "TestSscWriterMultiblock"; - adios2::Params engineParams = {{"Verbose", "0"}}; + { + std::string filename = "TestSscWriterMultiblock"; + adios2::Params engineParams = {{"Verbose", "0"}}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); - size_t steps = 100; + size_t steps = 100; - if (mpiGroup == 0) - { - Dims shape = {(size_t)mpiSize * 2, 10}; - Dims start = {(size_t)mpiRank * 2, 0}; - Dims count = {1, 10}; - Writer(shape, start, count, steps, engineParams, filename); - } + if (mpiGroup == 0) + { + Dims shape = {(size_t)mpiSize * 2, 10}; + Dims start = {(size_t)mpiRank * 2, 0}; + Dims count = {1, 10}; + Writer(shape, start, count, steps, engineParams, filename); + } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (mpiGroup == 1) - { - Dims shape = {(size_t)mpiSize * 2, 10}; - Dims start = {0, 0}; - Dims count = {(size_t)mpiSize * 2, 10}; - Reader(shape, start, count, steps, engineParams, filename); + if (mpiGroup == 1) + { + Dims shape = {(size_t)mpiSize * 2, 10}; + Dims start = {0, 0}; + Dims count = {(size_t)mpiSize * 2, 10}; + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); + { + std::string filename = "TestSscWriterMultiblockNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 100; + + if (mpiGroup == 0) + { + Dims shape = {(size_t)mpiSize * 2, 10}; + Dims start = {(size_t)mpiRank * 2, 0}; + Dims count = {1, 10}; + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + Dims shape = {(size_t)mpiSize * 2, 10}; + Dims start = {0, 0}; + Dims count = {(size_t)mpiSize * 2, 10}; + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); + } } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscXgc2Way.cpp b/testing/adios2/engine/ssc/TestSscXgc2Way.cpp index 8774526b90..23005a39be 100644 --- a/testing/adios2/engine/ssc/TestSscXgc2Way.cpp +++ b/testing/adios2/engine/ssc/TestSscXgc2Way.cpp @@ -130,57 +130,115 @@ void gene(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscXgc2Way) { - Dims start, count, shape; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - - if (worldSize < 6) { - return; + Dims start, count, shape; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + if (worldSize < 6) + { + return; + } + + if (worldRank < 2) + { + mpiGroup = 0; + } + else + { + mpiGroup = 1; + } + + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "2"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + xgc(shape, start, count, steps, engineParams); + } + + if (mpiGroup == 1) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "2"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + gene(shape, start, shape, steps, engineParams); + } + + MPI_Barrier(MPI_COMM_WORLD); } - if (worldRank < 2) - { - mpiGroup = 0; - } - else { - mpiGroup = 1; + Dims start, count, shape; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + if (worldSize < 6) + { + return; + } + + if (worldRank < 2) + { + mpiGroup = 0; + } + else + { + mpiGroup = 1; + } + + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "2"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + xgc(shape, start, count, steps, engineParams); + } + + if (mpiGroup == 1) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "2"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + gene(shape, start, shape, steps, engineParams); + } + + MPI_Barrier(MPI_COMM_WORLD); } - - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); - - size_t steps = 20; - - if (mpiGroup == 0) - { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "2"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - xgc(shape, start, count, steps, engineParams); - } - - if (mpiGroup == 1) - { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "2"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - gene(shape, start, shape, steps, engineParams); - } - - MPI_Barrier(MPI_COMM_WORLD); } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscXgc3Way.cpp b/testing/adios2/engine/ssc/TestSscXgc3Way.cpp index ecbccb576f..daa2c4510b 100644 --- a/testing/adios2/engine/ssc/TestSscXgc3Way.cpp +++ b/testing/adios2/engine/ssc/TestSscXgc3Way.cpp @@ -223,73 +223,146 @@ void gene(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscXgc3Way) { - Dims start, count, shape; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - - if (worldSize < 4) - { - return; - } - - if (worldRank == 0) - { - mpiGroup = 0; - } - else if (worldRank == 1) - { - mpiGroup = 1; - } - else - { - mpiGroup = 2; - } - - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); - - size_t steps = 20; - - if (mpiGroup == 0) { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "4"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - coupler(shape, start, count, steps, engineParams); + Dims start, count, shape; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + if (worldSize < 4) + { + return; + } + + if (worldRank == 0) + { + mpiGroup = 0; + } + else if (worldRank == 1) + { + mpiGroup = 1; + } + else + { + mpiGroup = 2; + } + + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + coupler(shape, start, count, steps, engineParams); + } + + if (mpiGroup == 1) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + gene(shape, start, shape, steps, engineParams); + } + + if (mpiGroup == 2) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + xgc(shape, start, count, steps, engineParams); + } + MPI_Barrier(MPI_COMM_WORLD); } - if (mpiGroup == 1) { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "4"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - gene(shape, start, shape, steps, engineParams); + Dims start, count, shape; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + if (worldSize < 4) + { + return; + } + + if (worldRank == 0) + { + mpiGroup = 0; + } + else if (worldRank == 1) + { + mpiGroup = 1; + } + else + { + mpiGroup = 2; + } + + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + coupler(shape, start, count, steps, engineParams); + } + + if (mpiGroup == 1) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + gene(shape, start, shape, steps, engineParams); + } + + if (mpiGroup == 2) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + xgc(shape, start, count, steps, engineParams); + } + MPI_Barrier(MPI_COMM_WORLD); } - - if (mpiGroup == 2) - { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "4"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - xgc(shape, start, count, steps, engineParams); - } - - MPI_Barrier(MPI_COMM_WORLD); } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp b/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp index df381ea4ab..ff050051a2 100644 --- a/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp +++ b/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp @@ -219,73 +219,148 @@ void gene(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscXgc3WayMatchedSteps) { - Dims start, count, shape; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - - if (worldSize < 4) - { - return; - } - - if (worldRank == 0) - { - mpiGroup = 0; - } - else if (worldRank == 1) - { - mpiGroup = 1; - } - else - { - mpiGroup = 2; - } - - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); - - size_t steps = 20; - - if (mpiGroup == 0) { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "4"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - coupler(shape, start, count, steps, engineParams); + Dims start, count, shape; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + if (worldSize < 4) + { + return; + } + + if (worldRank == 0) + { + mpiGroup = 0; + } + else if (worldRank == 1) + { + mpiGroup = 1; + } + else + { + mpiGroup = 2; + } + + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + coupler(shape, start, count, steps, engineParams); + } + + if (mpiGroup == 1) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + gene(shape, start, shape, steps, engineParams); + } + + if (mpiGroup == 2) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + xgc(shape, start, count, steps, engineParams); + } + + MPI_Barrier(MPI_COMM_WORLD); } - if (mpiGroup == 1) { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "4"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - gene(shape, start, shape, steps, engineParams); + Dims start, count, shape; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + if (worldSize < 4) + { + return; + } + + if (worldRank == 0) + { + mpiGroup = 0; + } + else if (worldRank == 1) + { + mpiGroup = 1; + } + else + { + mpiGroup = 2; + } + + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + coupler(shape, start, count, steps, engineParams); + } + + if (mpiGroup == 1) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + gene(shape, start, shape, steps, engineParams); + } + + if (mpiGroup == 2) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + xgc(shape, start, count, steps, engineParams); + } + + MPI_Barrier(MPI_COMM_WORLD); } - - if (mpiGroup == 2) - { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "4"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - xgc(shape, start, count, steps, engineParams); - } - - MPI_Barrier(MPI_COMM_WORLD); } int main(int argc, char **argv)