From 911a97c3c363e8c38b698498ee11704331e0b7e2 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Sat, 9 Sep 2023 07:30:21 -0400 Subject: [PATCH 1/2] Mod to keep MinBlocksInfo private to BP5Writer, not reuse Engine.h API. --- source/adios2/common/ADIOSTypes.cpp | 50 ++++++++++++++ source/adios2/common/ADIOSTypes.h | 3 + source/adios2/engine/bp5/BP5Writer.cpp | 27 +++++++- source/adios2/engine/bp5/BP5Writer.h | 2 + .../toolkit/format/bp5/BP5Serializer.cpp | 69 ++++++++++++++++++- .../adios2/toolkit/format/bp5/BP5Serializer.h | 4 +- source/adios2/toolkit/format/buffer/BufferV.h | 2 + .../toolkit/format/buffer/chunk/ChunkV.cpp | 24 +++++++ .../toolkit/format/buffer/chunk/ChunkV.h | 1 + .../toolkit/format/buffer/malloc/MallocV.cpp | 2 + .../toolkit/format/buffer/malloc/MallocV.h | 1 + 11 files changed, 181 insertions(+), 4 deletions(-) diff --git a/source/adios2/common/ADIOSTypes.cpp b/source/adios2/common/ADIOSTypes.cpp index dd8f1460af..bc507590fa 100644 --- a/source/adios2/common/ADIOSTypes.cpp +++ b/source/adios2/common/ADIOSTypes.cpp @@ -390,4 +390,54 @@ int TypeElementSize(DataType adiosvartype) } } +static void PrintMBI(std::ostream &os, const MinBlockInfo &blk, int Dims) +{ + os << "Writer: " << blk.WriterID << ", Blk: " << blk.BlockID << ", Start: {"; + if ((Dims == 0) || (blk.Start == NULL)) + os << "NULL"; + else + { + for (int i = 0; i < Dims; i++) + { + os << blk.Start[i]; + if (i < Dims - 1) + os << ", "; + } + } + os << "}, Count: {"; + + if ((Dims == 0) || (blk.Count == NULL)) + os << "NULL"; + else + { + for (int i = 0; i < Dims; i++) + { + os << blk.Count[i]; + if (i < Dims - 1) + os << ", "; + } + } + os << "}, Data: " << (void *)blk.BufferP << std::endl; +} + +void PrintMVI(std::ostream &os, const MinVarInfo &mvi) +{ + os << "Step: " << mvi.Step << " Dims: " << mvi.Dims << " Shape: {"; + if ((mvi.Dims == 0) || (mvi.Shape == NULL)) + os << "NULL"; + else + { + for (int i = 0; i < mvi.Dims; i++) + { + os << mvi.Shape[i]; + if (i < mvi.Dims - 1) + os << ", "; + } + } + os << "}, BlockCount: " << mvi.BlocksInfo.size() << " "; + for (const auto &blk : mvi.BlocksInfo) + PrintMBI(os, blk, mvi.Dims); + os << std::endl; +} + } // end namespace adios2 diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index 63a33d1a5c..7697c34624 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -214,6 +214,7 @@ struct MinBlockInfo MinMaxStruct MinMax; void *BufferP = NULL; }; + struct MinVarInfo { size_t Step; @@ -229,6 +230,8 @@ struct MinVarInfo } }; +void PrintMVI(std::ostream &os, const MinVarInfo &mvi); + // adios defaults #ifdef _WIN32 const std::string DefaultFileLibrary("fstream"); diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp index c0b6462f7a..6ee1bac5be 100644 --- a/source/adios2/engine/bp5/BP5Writer.cpp +++ b/source/adios2/engine/bp5/BP5Writer.cpp @@ -497,8 +497,6 @@ void BP5Writer::MarshalAttributes() void BP5Writer::EndStep() { - /* Seconds ts = Now() - m_EngineStart; - std::cout << "END STEP starts at: " << ts.count() << std::endl; */ m_BetweenStepPairs = false; PERFSTUBS_SCOPED_TIMER("BP5Writer::EndStep"); m_Profiler.Start("ES"); @@ -506,6 +504,26 @@ void BP5Writer::EndStep() m_Profiler.Start("ES_close"); MarshalAttributes(); +#ifdef NOT_DEF + const auto &vars = m_IO.GetVariables(); + for (const auto &varPair : vars) + { + auto baseVar = varPair.second.get(); + auto mvi = MinBlocksInfo(*baseVar); + if (mvi) + { + std::cout << "Info for Variable " << varPair.first << std::endl; + PrintMVI(std::cout, *mvi); + if (baseVar->m_Type == DataType::Double) + std::cout << "Double value is " << *((double *)mvi->BlocksInfo[0].BufferP) + << std::endl; + delete mvi; + } + else + std::cout << "Variable " << varPair.first << " not written on this step" << std::endl; + } +#endif + // true: advances step auto TSInfo = m_BP5Serializer.CloseTimestep((int)m_WriterStep, m_Parameters.AsyncWrite || m_Parameters.DirectIO); @@ -668,6 +686,11 @@ void BP5Writer::Init() InitBPBuffer(); } +MinVarInfo *BP5Writer::MinBlocksInfo(const core::VariableBase &Var) +{ + return m_BP5Serializer.MinBlocksInfo(Var); +} + void BP5Writer::InitParameters() { ParseParams(m_IO, m_Parameters); diff --git a/source/adios2/engine/bp5/BP5Writer.h b/source/adios2/engine/bp5/BP5Writer.h index 7d6ad47a04..454454c5cf 100644 --- a/source/adios2/engine/bp5/BP5Writer.h +++ b/source/adios2/engine/bp5/BP5Writer.h @@ -115,6 +115,8 @@ class BP5Writer : public BP5Engine, public core::Engine */ void NotifyEngineAttribute(std::string name, AttributeBase *Attr, void *data) noexcept; + MinVarInfo *MinBlocksInfo(const VariableBase &); + void EnterComputationBlock() noexcept; /** Inform about computation block through User->ADIOS->IO */ void ExitComputationBlock() noexcept; diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp index aad15ae794..8b10c30b46 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp @@ -80,7 +80,7 @@ void BP5Serializer::Init() ((BP5MetadataInfoStruct *)MetadataBuf)->BitField = (std::size_t *)malloc(sizeof(size_t)); ((BP5MetadataInfoStruct *)MetadataBuf)->DataBlockSize = 0; } -BP5Serializer::BP5WriterRec BP5Serializer::LookupWriterRec(void *Key) +BP5Serializer::BP5WriterRec BP5Serializer::LookupWriterRec(void *Key) const { for (int i = 0; i < Info.RecCount; i++) { @@ -858,6 +858,73 @@ void BP5Serializer::Marshal(void *Variable, const char *Name, const DataType Typ } } +MinVarInfo *BP5Serializer::MinBlocksInfo(const core::VariableBase &Var) +{ + BP5WriterRec VarRec = LookupWriterRec((void *)&Var); + + if (!VarRec) + return NULL; + + MinVarInfo *MV = new MinVarInfo((int)VarRec->DimCount, (size_t *)Var.m_Shape.data()); + + BP5MetadataInfoStruct *MBase = (struct BP5MetadataInfoStruct *)MetadataBuf; + + int AlreadyWritten = BP5BitfieldTest(MBase, VarRec->FieldID); + + if (!AlreadyWritten) + return MV; + + if (Var.m_SingleValue) + { + // single value case + MinBlockInfo Blk; + Blk.MinMax.Init(Var.m_Type); + Blk.WriterID = (int)-1; + Blk.BlockID = 0; + Blk.Start = NULL; + Blk.Count = NULL; + if (Var.m_Type != DataType::String) + { + Blk.BufferP = (char *)(MetadataBuf) + VarRec->MetaOffset; + } + else + { + char **StrPtr = (char **)((char *)(MetadataBuf) + VarRec->MetaOffset); + Blk.BufferP = *StrPtr; + } + MV->BlocksInfo.push_back(Blk); + } + else + { + // everything else + MetaArrayRec *MetaEntry = (MetaArrayRec *)((char *)(MetadataBuf) + VarRec->MetaOffset); + for (size_t b = 0; b < MetaEntry->BlockCount; b++) + { + MinBlockInfo Blk; + Blk.MinMax.Init(Var.m_Type); + Blk.WriterID = (int)-1; + Blk.BlockID = 0; + Blk.Start = NULL; + if (MetaEntry->Offsets) + { + Blk.Start = &(MetaEntry->Offsets[b * MetaEntry->Dims]); + } + Blk.Count = &(MetaEntry->Count[b * MetaEntry->Dims]); + if (MetaEntry->DataBlockLocation[b] < m_PriorDataBufferSizeTotal) + { + Blk.BufferP = (void *)(intptr_t)(-1); // data is out of memory + } + else + { + Blk.BufferP = CurDataBuffer->GetPtr(MetaEntry->DataBlockLocation[b] - + m_PriorDataBufferSizeTotal); + } + MV->BlocksInfo.push_back(Blk); + } + } + return MV; +} + void BP5Serializer::MarshalAttribute(const char *Name, const DataType Type, size_t ElemSize, size_t ElemCount, const void *Data) { diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.h b/source/adios2/toolkit/format/bp5/BP5Serializer.h index 14c5d3f6e1..1974972ad2 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.h +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.h @@ -127,6 +127,8 @@ class BP5Serializer : virtual public BP5Base size_t DebugGetDataBufferSize() const; + MinVarInfo *MinBlocksInfo(const core::VariableBase &Var); + int m_StatsLevel = 1; /* Variables to help appending to existing file */ @@ -204,7 +206,7 @@ class BP5Serializer : virtual public BP5Base size_t m_PriorDataBufferSizeTotal = 0; - BP5WriterRec LookupWriterRec(void *Key); + BP5WriterRec LookupWriterRec(void *Key) const; BP5WriterRec CreateWriterRec(void *Variable, const char *Name, DataType Type, size_t ElemSize, size_t DimCount); void ValidateWriterRec(BP5WriterRec Rec, void *Variable); diff --git a/source/adios2/toolkit/format/buffer/BufferV.h b/source/adios2/toolkit/format/buffer/BufferV.h index 0436ae363d..0ca1e3eac0 100644 --- a/source/adios2/toolkit/format/buffer/BufferV.h +++ b/source/adios2/toolkit/format/buffer/BufferV.h @@ -63,6 +63,8 @@ class BufferV virtual void *GetPtr(int bufferIdx, size_t posInBuffer) = 0; + virtual void *GetPtr(size_t overallPosInVector) = 0; + protected: std::vector zero; const bool m_AlwaysCopy = false; diff --git a/source/adios2/toolkit/format/buffer/chunk/ChunkV.cpp b/source/adios2/toolkit/format/buffer/chunk/ChunkV.cpp index 30fb427fe0..4f4f04fc05 100644 --- a/source/adios2/toolkit/format/buffer/chunk/ChunkV.cpp +++ b/source/adios2/toolkit/format/buffer/chunk/ChunkV.cpp @@ -255,6 +255,30 @@ void *ChunkV::GetPtr(int bufferIdx, size_t posInBuffer) } } +void *ChunkV::GetPtr(size_t OverallPosInBuffer) +{ + int bufferIdx = 0; + if (DataV.size() == 0) + return nullptr; + while (DataV[bufferIdx].Size <= OverallPosInBuffer) + { + OverallPosInBuffer -= DataV[bufferIdx].Size; + bufferIdx++; + if (static_cast(bufferIdx) > DataV.size()) + { + helper::Throw( + "Toolkit", "format::ChunkV", "GetPtr", + "ChunkV::GetPtr(" + std::to_string(OverallPosInBuffer) + + ") refers to a non-existing or deferred memory chunk."); + return nullptr; + } + } + if (DataV[bufferIdx].External) + return ((char *)DataV[bufferIdx].External) + OverallPosInBuffer; + + return (void *)((char *)DataV[bufferIdx].Base + OverallPosInBuffer); +} + std::vector ChunkV::DataVec() noexcept { std::vector iov(DataV.size()); diff --git a/source/adios2/toolkit/format/buffer/chunk/ChunkV.h b/source/adios2/toolkit/format/buffer/chunk/ChunkV.h index ef65c08665..fc129243fe 100644 --- a/source/adios2/toolkit/format/buffer/chunk/ChunkV.h +++ b/source/adios2/toolkit/format/buffer/chunk/ChunkV.h @@ -38,6 +38,7 @@ class ChunkV : public BufferV virtual void DownsizeLastAlloc(const size_t oldSize, const size_t newSize); virtual void *GetPtr(int bufferIdx, size_t posInBuffer); + virtual void *GetPtr(size_t OverallPosInBuffer); void CopyDataToBuffer(const size_t size, const void *buf, size_t pos, MemorySpace MemSpace); diff --git a/source/adios2/toolkit/format/buffer/malloc/MallocV.cpp b/source/adios2/toolkit/format/buffer/malloc/MallocV.cpp index 2ba21dca80..90359071fb 100644 --- a/source/adios2/toolkit/format/buffer/malloc/MallocV.cpp +++ b/source/adios2/toolkit/format/buffer/malloc/MallocV.cpp @@ -157,6 +157,8 @@ void *MallocV::GetPtr(int bufferIdx, size_t posInBuffer) } } +void *MallocV::GetPtr(size_t posInBuffer) { return m_InternalBlock + posInBuffer; } + std::vector MallocV::DataVec() noexcept { std::vector iov(DataV.size()); diff --git a/source/adios2/toolkit/format/buffer/malloc/MallocV.h b/source/adios2/toolkit/format/buffer/malloc/MallocV.h index 6553271099..eba6ba9d09 100644 --- a/source/adios2/toolkit/format/buffer/malloc/MallocV.h +++ b/source/adios2/toolkit/format/buffer/malloc/MallocV.h @@ -42,6 +42,7 @@ class MallocV : public BufferV void DownsizeLastAlloc(const size_t oldSize, const size_t newSize); virtual void *GetPtr(int bufferIdx, size_t posInBuffer); + virtual void *GetPtr(size_t posInBuffer); private: char *m_InternalBlock = NULL; From 69ac221eefec73239c51d289da78dd4b44fd0b58 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Sat, 9 Sep 2023 07:53:58 -0400 Subject: [PATCH 2/2] Mod not to overload to make some compilers happy --- source/adios2/engine/bp5/BP5Writer.cpp | 4 ++-- source/adios2/engine/bp5/BP5Writer.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp index 6ee1bac5be..465a44d959 100644 --- a/source/adios2/engine/bp5/BP5Writer.cpp +++ b/source/adios2/engine/bp5/BP5Writer.cpp @@ -509,7 +509,7 @@ void BP5Writer::EndStep() for (const auto &varPair : vars) { auto baseVar = varPair.second.get(); - auto mvi = MinBlocksInfo(*baseVar); + auto mvi = WriterMinBlocksInfo(*baseVar); if (mvi) { std::cout << "Info for Variable " << varPair.first << std::endl; @@ -686,7 +686,7 @@ void BP5Writer::Init() InitBPBuffer(); } -MinVarInfo *BP5Writer::MinBlocksInfo(const core::VariableBase &Var) +MinVarInfo *BP5Writer::WriterMinBlocksInfo(const core::VariableBase &Var) { return m_BP5Serializer.MinBlocksInfo(Var); } diff --git a/source/adios2/engine/bp5/BP5Writer.h b/source/adios2/engine/bp5/BP5Writer.h index 454454c5cf..72d0602217 100644 --- a/source/adios2/engine/bp5/BP5Writer.h +++ b/source/adios2/engine/bp5/BP5Writer.h @@ -115,7 +115,7 @@ class BP5Writer : public BP5Engine, public core::Engine */ void NotifyEngineAttribute(std::string name, AttributeBase *Attr, void *data) noexcept; - MinVarInfo *MinBlocksInfo(const VariableBase &); + MinVarInfo *WriterMinBlocksInfo(const VariableBase &); void EnterComputationBlock() noexcept; /** Inform about computation block through User->ADIOS->IO */