Skip to content

Commit

Permalink
Merge pull request #2989 from eisenhauer/MoreTests
Browse files Browse the repository at this point in the history 
More BP5 Tests
  • Loading branch information
@eisenhauer
eisenhauer authored Jan 4, 2022
2 parents ff8326c + e9693a7 commit c8432b3
Show file tree
Hide file tree
Showing 10 changed files with 97 additions and 28 deletions.
5 changes: 5 additions & 0 deletions source/adios2/engine/bp5/BP5Writer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1212,6 +1212,11 @@ void BP5Writer::PopulateMetadataIndexFileContent(
position += 8;
}

size_t BP5Writer::DebugGetDataBufferSize() const
{
return m_BP5Serializer.DebugGetDataBufferSize();
}

} // end namespace engine
} // end namespace core
} // end namespace adios2
2 changes: 2 additions & 0 deletions source/adios2/engine/bp5/BP5Writer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ class BP5Writer : public BP5Engine, public core::Engine
void EndStep() final;
void Flush(const int transportIndex = -1) final;

size_t DebugGetDataBufferSize() const final;

private:
/** Single object controlling BP buffering */
format::BP5Serializer m_BP5Serializer;
Expand Down
19 changes: 12 additions & 7 deletions source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -859,15 +859,15 @@ bool BP5Deserializer::QueueGet(core::VariableBase &variable, void *DestData)
}
}

void BP5Deserializer::GetSingleValueFromMetadata(core::VariableBase &variable,
bool BP5Deserializer::GetSingleValueFromMetadata(core::VariableBase &variable,
BP5VarRec *VarRec,
void *DestData, size_t Step,
size_t WriterRank)
{
char *src = (char *)GetMetadataBase(VarRec, Step, WriterRank);

if (!src)
return;
return false;

if (variable.m_SelectionType == adios2::SelectionType::WriteBlock)
WriterRank = variable.m_BlockID;
Expand All @@ -881,6 +881,7 @@ void BP5Deserializer::GetSingleValueFromMetadata(core::VariableBase &variable,
std::string *TmpStr = static_cast<std::string *>(DestData);
TmpStr->assign(*(const char **)src);
}
return true;
}

bool BP5Deserializer::QueueGetSingle(core::VariableBase &variable,
Expand All @@ -889,9 +890,13 @@ bool BP5Deserializer::QueueGetSingle(core::VariableBase &variable,
BP5VarRec *VarRec = VarByKey[&variable];
if (VarRec->OrigShapeID == ShapeID::GlobalValue)
{
int WriterRank = 0;
GetSingleValueFromMetadata(variable, VarRec, DestData, Step,
WriterRank);
for (size_t WriterRank = 0; WriterRank < m_WriterCohortSize;
WriterRank++)
{
if (GetSingleValueFromMetadata(variable, VarRec, DestData, Step,
WriterRank))
return false;
}
return false;
}
if (VarRec->OrigShapeID == ShapeID::LocalValue)
Expand All @@ -902,8 +907,8 @@ bool BP5Deserializer::QueueGetSingle(core::VariableBase &variable,
WriterRank < variable.m_Count[0] + variable.m_Start[0];
WriterRank++)
{
GetSingleValueFromMetadata(variable, VarRec, DestData, Step,
WriterRank);
(void)GetSingleValueFromMetadata(variable, VarRec, DestData, Step,
WriterRank);
DestData = (char *)DestData +
variable.m_ElementSize; // use variable.m_ElementSize
// because it's the size in local
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/toolkit/format/bp5/BP5Deserializer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ class BP5Deserializer : virtual public BP5Base
const size_t *LocalOffsets, size_t *LocalIndex);
size_t RelativeToAbsoluteStep(const BP5VarRec *VarRec, size_t RelStep);
int FindOffset(size_t Dims, const size_t *Size, const size_t *Index);
void GetSingleValueFromMetadata(core::VariableBase &variable,
bool GetSingleValueFromMetadata(core::VariableBase &variable,
BP5VarRec *VarRec, void *DestData,
size_t Step, size_t WriterRank);
void ExtractSelectionFromPartialRM(int ElementSize, size_t Dims,
Expand Down
7 changes: 7 additions & 0 deletions source/adios2/toolkit/format/bp5/BP5Serializer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1144,5 +1144,12 @@ void *BP5Serializer::GetPtr(int bufferIdx, size_t posInBuffer)
return CurDataBuffer->GetPtr(bufferIdx, posInBuffer);
}

size_t BP5Serializer::DebugGetDataBufferSize() const
{
if (CurDataBuffer == NULL)
return 0;
return CurDataBuffer->Size();
}

} // end namespace format
} // end namespace adios2
2 changes: 2 additions & 0 deletions source/adios2/toolkit/format/bp5/BP5Serializer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ class BP5Serializer : virtual public BP5Base
void *GetPtr(int bufferIdx, size_t posInBuffer);
size_t CalcSize(const size_t Count, const size_t *Vals);

size_t DebugGetDataBufferSize() const;

int m_StatsLevel = 1;

private:
Expand Down
6 changes: 3 additions & 3 deletions testing/adios2/engine/bp/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ bp3_bp4_gtest_add_tests_helper(WriteFlushRead MPI_ALLOW)
bp3_bp4_gtest_add_tests_helper(WriteMultiblockRead MPI_ALLOW)
bp3_bp4_gtest_add_tests_helper(WriteReadMultiblock MPI_ALLOW)
bp_gtest_add_tests_helper(WriteReadVector MPI_ALLOW)
bp3_bp4_gtest_add_tests_helper(WriteReadAttributesMultirank MPI_ALLOW)
bp_gtest_add_tests_helper(WriteReadAttributesMultirank MPI_ALLOW)
bp_gtest_add_tests_helper(LargeMetadata MPI_ALLOW)
if(ADIOS2_HAVE_BP5)
set (BP5LargeMeta "Engine.BP.BPLargeMetadata.BPWrite1D_LargeMetadata.BP5.Serial")
Expand All @@ -96,13 +96,13 @@ bp3_bp4_gtest_add_tests_helper(TimeAggregation MPI_ALLOW)
bp_gtest_add_tests_helper(NoXMLRecovery MPI_ALLOW)
bp3_bp4_gtest_add_tests_helper(StepsFileGlobalArray MPI_ALLOW)
bp3_bp4_gtest_add_tests_helper(StepsFileLocalArray MPI_ALLOW)
bp3_bp4_gtest_add_tests_helper(SelectSteps MPI_ALLOW)
bp_gtest_add_tests_helper(SelectSteps MPI_ALLOW)

bp3_bp4_gtest_add_tests_helper(SelectionsOnRowMajorData MPI_NONE)
bp3_bp4_gtest_add_tests_helper(SelectionsOnColumnMajorData MPI_NONE)

if(NOT MSVC)
bp3_bp4_gtest_add_tests_helper(BufferSize MPI_NONE)
bp_gtest_add_tests_helper(BufferSize MPI_NONE)
endif()

if(ADIOS2_HAVE_MPI)
Expand Down
52 changes: 45 additions & 7 deletions testing/adios2/engine/bp/TestBPStepsFileGlobalArray.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,13 +159,13 @@ TEST_P(BPStepsFileGlobalArrayReaders, EveryStep)
{
io.SetEngine(engineName);
}
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (readMode == ReadMode::ReadFileAll)
{
/// Read back data with File reading mode
/// Read back the whole thing and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout << "Read with File reading mode, read all steps at once"
Expand Down Expand Up @@ -210,6 +210,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, EveryStep)
{
/// Read back data with File reading mode
/// Read back step by step and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout << "Read with File reading mode, read step by step"
Expand Down Expand Up @@ -239,6 +242,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, EveryStep)
{
/// Read back data with File reading mode
/// Read back step by step and block by block and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout << "Read with File reading mode, read step by step, "
Expand Down Expand Up @@ -274,6 +280,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, EveryStep)
{
/// Read back data with Stream reading mode
/// Read back step by step and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout << "Read with Stream reading mode, read step by step"
Expand Down Expand Up @@ -302,6 +311,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, EveryStep)
}
else if (readMode == ReadMode::ReadStreamBlocks)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back data with Stream reading mode
/// Read back step by step and check data
if (!mpiRank)
Expand Down Expand Up @@ -412,11 +424,12 @@ TEST_P(BPStepsFileGlobalArrayReaders, NewVarPerStep)
{
io.SetEngine(engineName);
}
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (readMode == ReadMode::ReadFileAll)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back each variable with File reading mode
/// Use SetStepSelection(0,1) explicitly
if (!mpiRank)
Expand Down Expand Up @@ -459,6 +472,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, NewVarPerStep)
}
else if (readMode == ReadMode::ReadFileStepByStep)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back each variable with File reading mode
/// and do not use SetStepSelection() so default read after open is
/// tested
Expand Down Expand Up @@ -490,6 +506,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, NewVarPerStep)
}
else if (readMode == ReadMode::ReadFileStepByStepBlocks)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back each variable with File reading mode
/// Read back block by block and check data
if (!mpiRank)
Expand Down Expand Up @@ -527,6 +546,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, NewVarPerStep)
}
else if (readMode == ReadMode::ReadStream)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

/// Read back each variable with Streaming reading mode
if (!mpiRank)
{
Expand Down Expand Up @@ -561,6 +583,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, NewVarPerStep)
}
else if (readMode == ReadMode::ReadStreamBlocks)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

/// Read back each variable with Streaming reading mode
if (!mpiRank)
{
Expand Down Expand Up @@ -699,11 +724,12 @@ TEST_P(BPStepsFileGlobalArrayParameters, EveryOtherStep)
{
io.SetEngine(engineName);
}
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (readMode == ReadMode::ReadFileAll)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back data with File reading mode
/// Read back the whole thing and check data
if (!mpiRank)
Expand Down Expand Up @@ -736,6 +762,9 @@ TEST_P(BPStepsFileGlobalArrayParameters, EveryOtherStep)
}
else if (readMode == ReadMode::ReadFileStepByStep)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back data with File reading mode
/// Read back step by step and check data
if (!mpiRank)
Expand Down Expand Up @@ -767,6 +796,9 @@ TEST_P(BPStepsFileGlobalArrayParameters, EveryOtherStep)
}
else if (readMode == ReadMode::ReadFileStepByStepBlocks)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back data with File reading mode
/// Read back step by step, block by block and check data
if (!mpiRank)
Expand Down Expand Up @@ -804,6 +836,9 @@ TEST_P(BPStepsFileGlobalArrayParameters, EveryOtherStep)
}
else if (readMode == ReadMode::ReadStream)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

/// Read back data with Stream reading mode
/// Read back step by step and check data
if (!mpiRank)
Expand Down Expand Up @@ -845,6 +880,9 @@ TEST_P(BPStepsFileGlobalArrayParameters, EveryOtherStep)
}
else if (readMode == ReadMode::ReadStreamBlocks)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

/// Read back data with Stream reading mode
/// Read back step by step and check data
if (!mpiRank)
Expand Down
27 changes: 18 additions & 9 deletions testing/adios2/engine/bp/TestBPStepsFileLocalArray.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,13 +151,13 @@ TEST_P(BPStepsFileLocalArrayReaders, EveryStep)
{
io.SetEngine(engineName);
}
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (readMode == ReadMode::ReadFileStepByStepBlocks)
{
/// Read back data with File reading mode
/// Read back step by step and block by block and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout << "Read with File reading mode, read step by step, "
Expand Down Expand Up @@ -194,6 +194,9 @@ TEST_P(BPStepsFileLocalArrayReaders, EveryStep)
{
/// Read back data with Stream reading mode
/// Read back step by step and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout << "Read with Stream reading mode, read step by step, "
Expand Down Expand Up @@ -303,13 +306,13 @@ TEST_P(BPStepsFileLocalArrayReaders, NewVarPerStep)
{
io.SetEngine(engineName);
}
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (readMode == ReadMode::ReadFileStepByStepBlocks)
{
/// Read back each variable with File reading mode
/// Read back block by block and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout
Expand Down Expand Up @@ -347,6 +350,9 @@ TEST_P(BPStepsFileLocalArrayReaders, NewVarPerStep)
else if (readMode == ReadMode::ReadStreamBlocks)
{
/// Read back each variable with Streaming reading mode
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout
Expand Down Expand Up @@ -482,11 +488,11 @@ TEST_P(BPStepsFileLocalArrayParameters, EveryOtherStep)
{
io.SetEngine(engineName);
}
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (readMode == ReadMode::ReadFileStepByStepBlocks)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back data with File reading mode
/// Read back step by step, block by block and check data
if (!mpiRank)
Expand Down Expand Up @@ -527,6 +533,9 @@ TEST_P(BPStepsFileLocalArrayParameters, EveryOtherStep)
{
/// Read back data with Stream reading mode
/// Read back step by step and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout
Expand Down
3 changes: 2 additions & 1 deletion testing/adios2/engine/bp/TestBPWriteReadAttributesMultirank.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,8 @@ TEST_F(BPWriteReadAttributeTestMultirank, ADIOS2BPWriteReadArrayTypes)
ioRead.SetEngine(engineName);
}

adios2::Engine bpRead = ioRead.Open(fName, adios2::Mode::Read);
adios2::Engine bpRead =
ioRead.Open(fName, adios2::Mode::ReadRandomAccess);

auto var = ioRead.InquireVariable<int>(varpath);
EXPECT_TRUE(var);
Expand Down

0 comments on commit c8432b3

Please sign in to comment.