Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Implement AbsoluteSteps for BP5 Engine/serializer #3033

Merged
merged 2 commits into from
Feb 2, 2022
Merged

Implement AbsoluteSteps for BP5 Engine/serializer #3033

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ebe21e4
Implement AbsoluteSteps for BP5 Engine/serializer
eisenhauer Feb 1, 2022
fafef48
Kill unused variants of DoGetAbsoluteSteps
eisenhauer Feb 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions source/adios2/core/Engine.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,13 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
return nullptr; \
}

void Engine::DoGetAbsoluteSteps(const VariableBase &variable,
std::vector<size_t> &keys) const
{
ThrowUp("DoGetAbsoluteSteps");
return;
}

ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
#undef declare_type

Expand All @@ -165,12 +172,6 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
{ \
ThrowUp("DoBlocksInfo"); \
return std::vector<typename Variable<T>::BPInfo>(); \
} \
std::vector<size_t> Engine::DoGetAbsoluteSteps( \
const Variable<T> &variable) const \
{ \
ThrowUp("DoGetAbsoluteSteps"); \
return std::vector<size_t>(); \
}

ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
Expand Down
7 changes: 4 additions & 3 deletions source/adios2/core/Engine.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -546,9 +546,7 @@ class Engine
DoAllRelativeStepsBlocksInfo(const Variable<T> &variable) const; \
\
virtual std::vector<typename Variable<T>::BPInfo> DoBlocksInfo( \
const Variable<T> &variable, const size_t step) const; \
virtual std::vector<size_t> DoGetAbsoluteSteps( \
const Variable<T> &variable) const;
const Variable<T> &variable, const size_t step) const;

ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
#undef declare_type
Expand All @@ -563,6 +561,9 @@ class Engine

virtual size_t DoSteps() const;

virtual void DoGetAbsoluteSteps(const VariableBase &variable,
std::vector<size_t> &keys) const;

/** true: No more definitions or changes to existing variables are allowed
*/
bool m_WriterDefinitionsLocked = false;
Expand Down
5 changes: 5 additions & 0 deletions source/adios2/core/Engine.tcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,11 @@ std::vector<size_t> Engine::GetAbsoluteSteps(const Variable<T> &variable) const
{
const auto &m = variable.m_AvailableStepBlockIndexOffsets;
std::vector<size_t> keys;
if (m.size() == 0)
{
DoGetAbsoluteSteps(variable, keys);
return keys;
}
keys.reserve(m.size());
for (auto it = m.cbegin(); it != m.cend(); ++it)
{
Expand Down
7 changes: 7 additions & 0 deletions source/adios2/engine/bp5/BP5Reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,13 @@ void BP5Reader::ParseMetadataIndex(format::BufferSTL &bufferSTL,
} while (!oneStepOnly && position < buffer.size());
}

void BP5Reader::DoGetAbsoluteSteps(const VariableBase &variable,
std::vector<size_t> &keys) const
{
m_BP5Deserializer->GetAbsoluteSteps(variable, keys);
return;
}

#define declare_type(T) \
void BP5Reader::DoGetSync(Variable<T> &variable, T *data) \
{ \
Expand Down
3 changes: 3 additions & 0 deletions source/adios2/engine/bp5/BP5Reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,9 @@ class BP5Reader : public BP5Engine, public Engine

size_t DoSteps() const final;

void DoGetAbsoluteSteps(const VariableBase &variable,
std::vector<size_t> &keys) const final;

uint32_t m_WriterColumnMajor = 0;
bool m_ReaderIsRowMajor = true;
bool m_WriterIsRowMajor = true;
Expand Down
4 changes: 2 additions & 2 deletions source/adios2/toolkit/format/bp5/BP5Base.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ namespace adios2
namespace format
{

void BP5Base::BP5BitfieldSet(struct BP5MetadataInfoStruct *MBase, int Bit)
void BP5Base::BP5BitfieldSet(struct BP5MetadataInfoStruct *MBase, int Bit) const
{
size_t Element = Bit / (sizeof(size_t) * 8);
int ElementBit = Bit % (sizeof(size_t) * 8);
Expand All @@ -38,7 +38,7 @@ void BP5Base::BP5BitfieldSet(struct BP5MetadataInfoStruct *MBase, int Bit)
MBase->BitField[Element] |= ((size_t)1 << ElementBit);
}

int BP5Base::BP5BitfieldTest(struct BP5MetadataInfoStruct *MBase, int Bit)
int BP5Base::BP5BitfieldTest(struct BP5MetadataInfoStruct *MBase, int Bit) const
{
size_t Element = Bit / (sizeof(size_t) * 8);
int ElementBit = Bit % (sizeof(size_t) * 8);
Expand Down
4 changes: 2 additions & 2 deletions source/adios2/toolkit/format/bp5/BP5Base.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ class BP5Base
size_t DataBlockSize;
};

void BP5BitfieldSet(struct BP5MetadataInfoStruct *MBase, int Bit);
int BP5BitfieldTest(struct BP5MetadataInfoStruct *MBase, int Bit);
void BP5BitfieldSet(struct BP5MetadataInfoStruct *MBase, int Bit) const;
int BP5BitfieldTest(struct BP5MetadataInfoStruct *MBase, int Bit) const;
};
} // end namespace format
} // end namespace adios2
Expand Down
33 changes: 28 additions & 5 deletions source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,10 +202,10 @@ void BP5Deserializer::BreakdownArrayName(const char *Name, char **base_name_p,
*(rindex(*base_name_p, '_')) = 0;
}

BP5Deserializer::BP5VarRec *BP5Deserializer::LookupVarByKey(void *Key)
BP5Deserializer::BP5VarRec *BP5Deserializer::LookupVarByKey(void *Key) const
{
auto ret = VarByKey[Key];
return ret;
auto ret = VarByKey.find(Key);
return ret->second;
}

BP5Deserializer::BP5VarRec *BP5Deserializer::LookupVarByName(const char *Name)
Expand Down Expand Up @@ -464,7 +464,7 @@ void BP5Deserializer::SetupForStep(size_t Step, size_t WriterCount)
}
}

size_t BP5Deserializer::WriterCohortSize(size_t Step)
size_t BP5Deserializer::WriterCohortSize(size_t Step) const
{
if (m_RandomAccessMode)
{
Expand Down Expand Up @@ -1556,7 +1556,7 @@ BP5Deserializer::~BP5Deserializer()
}

void *BP5Deserializer::GetMetadataBase(BP5VarRec *VarRec, size_t Step,
size_t WriterRank)
size_t WriterRank) const
{
MetaArrayRec *writer_meta_base = NULL;
if (m_RandomAccessMode)
Expand Down Expand Up @@ -1828,6 +1828,29 @@ size_t BP5Deserializer::RelativeToAbsoluteStep(const BP5VarRec *VarRec,
return AbsStep;
}

void BP5Deserializer::GetAbsoluteSteps(const VariableBase &Var,
std::vector<size_t> &keys) const
{
BP5VarRec *VarRec = LookupVarByKey((void *)&Var);
if (!m_RandomAccessMode)
return;

for (size_t Step = 0; Step < m_ControlArray.size(); Step++)
{
for (size_t WriterRank = 0; WriterRank < WriterCohortSize(Step);
WriterRank++)
{
MetaArrayRec *writer_meta_base =
(MetaArrayRec *)GetMetadataBase(VarRec, Step, WriterRank);
if (writer_meta_base)
{
keys.push_back(Step);
break;
}
}
}
}

bool BP5Deserializer::VariableMinMax(const VariableBase &Var, const size_t Step,
MinMaxStruct &MinMax)
{
Expand Down
9 changes: 6 additions & 3 deletions source/adios2/toolkit/format/bp5/BP5Deserializer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ class BP5Deserializer : virtual public BP5Base
MinVarInfo *MinBlocksInfo(const VariableBase &Var, const size_t Step);
bool VariableMinMax(const VariableBase &var, const size_t Step,
MinMaxStruct &MinMax);
void GetAbsoluteSteps(const VariableBase &variable,
std::vector<size_t> &keys) const;

const bool m_WriterIsRowMajor;
const bool m_ReaderIsRowMajor;
Expand Down Expand Up @@ -133,7 +135,7 @@ class BP5Deserializer : virtual public BP5Base
// return the number of writers
// m_CurrentWriterCohortSize in streaming mode
// m_WriterCohortSize[Step] in random access mode
size_t WriterCohortSize(size_t Step);
size_t WriterCohortSize(size_t Step) const;

size_t m_LastAttrStep = MaxSizeT; // invalid timestep for start

Expand All @@ -157,7 +159,7 @@ class BP5Deserializer : virtual public BP5Base
bool NameIndicatesArray(const char *Name);
bool NameIndicatesAttrArray(const char *Name);
DataType TranslateFFSType2ADIOS(const char *Type, int size);
BP5VarRec *LookupVarByKey(void *Key);
BP5VarRec *LookupVarByKey(void *Key) const;
BP5VarRec *LookupVarByName(const char *Name);
BP5VarRec *CreateVarRec(const char *ArrayName);
void ReverseDimensions(size_t *Dimensions, int count, int times);
Expand Down Expand Up @@ -212,7 +214,8 @@ class BP5Deserializer : virtual public BP5Base
};
std::vector<BP5ArrayRequest> PendingRequests;
bool NeedWriter(BP5ArrayRequest Req, size_t i, size_t &NodeFirst);
void *GetMetadataBase(BP5VarRec *VarRec, size_t Step, size_t WriterRank);
void *GetMetadataBase(BP5VarRec *VarRec, size_t Step,
size_t WriterRank) const;
size_t CurTimestep = 0;
};

Expand Down