Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added support to query local arrays #3824

Closed
wants to merge 45 commits into from
Closed
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
f332881
1) removed unused code sections
guj Sep 23, 2023
1e96b21
clang-format v16
guj Sep 23, 2023
7066902
older clang-format
guj Sep 23, 2023
79528bd
removed extra ;
guj Sep 23, 2023
2149fe7
modified test. it failed because I consolidate block when all subblo…
guj Sep 24, 2023
2884f71
Import tests from bp to staging common, implement memory selection in…
eisenhauer Sep 22, 2023
51f7b6f
Try always using the MPI version
eisenhauer Sep 26, 2023
5bde490
Try always using the MPI version
eisenhauer Sep 27, 2023
c503940
Merge pull request #3823 from eisenhauer/SstMemSel
eisenhauer Sep 27, 2023
700839d
Add prototype testing of remote functionality (#3830)
eisenhauer Sep 30, 2023
107164d
Tweak Remote class and test multi-threaded file remote access (#3834)
eisenhauer Oct 14, 2023
ac07185
MPI: add timeout for conf test for MPI_DP (#3848)
vicentebolea Oct 17, 2023
8707f2b
Update test.cpp
guj Oct 17, 2023
d0fcf10
cmake: update minimum cmake to 3.12 (#3849)
vicentebolea Oct 17, 2023
975e8de
MPI_DP: do not call MPI_Init (#3847)
vicentebolea Oct 17, 2023
897ca42
Improve existing examples
spyridon97 Sep 14, 2023
3bdc1aa
Add examples from ADIOS2-Examples
spyridon97 Sep 18, 2023
fdee65d
Fix formatting issue in DetectOptions.cmake
spyridon97 Sep 19, 2023
b881acc
Remove testing based on ADIOS2-examples
spyridon97 Sep 19, 2023
bf63c65
Disable BUILD_TESTING and ADIOS2_BUILD_EXAMPLES by default
spyridon97 Sep 21, 2023
2425033
Add/Improve the ReadMe.md files in examples directory
spyridon97 Sep 21, 2023
326bb54
ci: enable tau check
vicentebolea Oct 16, 2023
2324daf
cmake: include ctest before detectoptions
vicentebolea Oct 17, 2023
861ccb7
Fix destdir install test (#3850)
vicentebolea Oct 18, 2023
a43d013
Merge pull request #3820 from spyridon97/organize-and-add-examples
spyridon97 Oct 18, 2023
6f33d87
Blosc2 USE ON: Fix Module Fallback (#3774)
ax3l Oct 19, 2023
2f581ec
BP5Serializer: handle local variables that use operators (#3859)
caitlinross Oct 20, 2023
3662939
Examples: Add BeginStep/EndStep wherever it was missing
spyridon97 Oct 11, 2023
e13cc18
inlineMWE example: Close files at the end
spyridon97 Oct 11, 2023
e98f442
Examples: Use BPFile instead of BP3/4/5 for future-proof
spyridon97 Oct 11, 2023
5315ead
Update the bpWriterReadHip example's cmake to run on crusher
spyridon97 Oct 19, 2023
2d4af78
Merge pull request #3856 from spyridon97/update-examples
spyridon97 Oct 20, 2023
2426600
bug fix: syntax error in json output (#3857)
guj Oct 20, 2023
347d356
Remove Slack as a contact mechanism (#3866)
eisenhauer Oct 20, 2023
afdef44
ascent,ci: remove unshallow flag
vicentebolea Oct 20, 2023
432c6ab
Server Improvements (#3862)
eisenhauer Oct 21, 2023
e53a1bf
Fix hip example compilation error (#3865)
spyridon97 Oct 21, 2023
de0ad6e
add changes to the query test file (someone moved to a new dir)
guj Oct 23, 2023
7322c8d
1) removed unused code sections
guj Sep 23, 2023
7402c20
clang-format v16
guj Sep 23, 2023
04a3dbc
older clang-format
guj Sep 23, 2023
75c34a1
removed extra ;
guj Sep 23, 2023
7c21614
modified test. it failed because I consolidate block when all subblo…
guj Sep 24, 2023
a5866d2
touched
guj Oct 23, 2023
a88271b
Merge branch 'queryLocalArray' of https://github.com/guj/ADIOS2 into …
guj Oct 23, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion bindings/CXX11/adios2/cxx11/Query.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ QueryWorker::QueryWorker(const std::string &configFile, adios2::Engine &reader)
delete m;
}

void QueryWorker::GetResultCoverage(std::vector<size_t> &touched_blockIDs)
{
m_Worker->GetResultCoverage(touched_blockIDs);
}

void QueryWorker::GetResultCoverage(std::vector<adios2::Box<adios2::Dims>> &touched_blocks)
{
adios2::Box<adios2::Dims> empty;
Expand All @@ -26,4 +31,5 @@ void QueryWorker::GetResultCoverage(const adios2::Box<adios2::Dims> &outputSelec
if (m_Worker)
return m_Worker->GetResultCoverage(outputSelection, touched_blocks);
}
}

} // namespace
1 change: 1 addition & 0 deletions bindings/CXX11/adios2/cxx11/Query.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ class QueryWorker
// configFile has query, can be either xml or json
QueryWorker(const std::string &configFile, adios2::Engine &engine);

void GetResultCoverage(std::vector<size_t> &touched_block_ids);
// touched_blocks is a list of regions specified by (start, count),
// that contains data that satisfies the query file
void GetResultCoverage(std::vector<adios2::Box<adios2::Dims>> &touched_blocks);
Expand Down
7 changes: 7 additions & 0 deletions bindings/Python/py11Query.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -36,5 +36,12 @@ std::vector<Box<Dims>> Query::GetResult()
return touched_blocks;
}

std::vector<size_t> Query::GetBlockIDs()
{
std::vector<size_t> touched_block_ids;
m_QueryWorker->GetResultCoverage(touched_block_ids);
return touched_block_ids;
}

} // py11
} // adios2
1 change: 1 addition & 0 deletions bindings/Python/py11Query.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ class Query
explicit operator bool() const noexcept;

std::vector<Box<Dims>> GetResult();
std::vector<size_t> GetBlockIDs();

private:
Query(adios2::query::Worker *qw);
Expand Down
3 changes: 2 additions & 1 deletion bindings/Python/py11glue.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,8 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
"adios2 query construction, a xml query File and a read engine",
pybind11::arg("queryFile"), pybind11::arg("reader") = true)

.def("GetResult", &adios2::py11::Query::GetResult);
.def("GetResult", &adios2::py11::Query::GetResult)
.def("GetBlockIDs", &adios2::py11::Query::GetBlockIDs);

pybind11::class_<adios2::py11::Variable>(m, "Variable")
// Python 2
Expand Down
83 changes: 36 additions & 47 deletions examples/query/test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,29 @@
#include <string>
#include <vector>

// #include "adios2/toolkit/query/Worker.h"
// returns block ids
guj marked this conversation as resolved.
Show resolved Hide resolved
void queryIDs(adios2::IO &queryIO, std::string &dataFileName, std::string &queryFile)
{
adios2::Engine reader = queryIO.Open(dataFileName, adios2::Mode::Read, MPI_COMM_WORLD);
// adios2::QueryWorker* worker = NULL;
queryIO.SetParameter("StreamReader", "true");
std::vector<size_t> touched_blockIDs;

while (reader.BeginStep() == adios2::StepStatus::OK)
{
adios2::QueryWorker w = adios2::QueryWorker(queryFile, reader);
w.GetResultCoverage(touched_blockIDs);

std::cout << " Num touched blocks =" << touched_blockIDs.size() << std::endl;
for (auto n : touched_blockIDs)
{
std::cout << "\t[" << n << "] " << std::endl;
}

reader.EndStep();
}
reader.Close();
}

void queryWithStreaming(adios2::IO &queryIO, std::string &dataFileName, std::string &queryFile)
{
Expand All @@ -24,7 +46,10 @@ void queryWithStreaming(adios2::IO &queryIO, std::string &dataFileName, std::str
adios2::QueryWorker w = adios2::QueryWorker(queryFile, reader);
w.GetResultCoverage(touched_blocks);

std::cout << " ... now can read out touched blocks ... size=" << touched_blocks.size()
std::cout << " Num touched regions ="
<< touched_blocks.size()
// std::cout << " ... now can read out touched blocks ... size=" <<
// touched_blocks.size()
<< std::endl;
for (auto n : touched_blocks)
{
Expand Down Expand Up @@ -68,12 +93,6 @@ int main(int argc, char *argv[])
configFileName = argv[1];
dataFileName = argv[2];

if (rank == 0)
{
std::cout << " using config file = " << configFileName << std::endl;
std::cout << " data file = " << dataFileName << std::endl;
}

adios2::ADIOS ad = adios2::ADIOS(configFileName, MPI_COMM_WORLD);

adios2::IO queryIO = ad.DeclareIO("query");
Expand All @@ -83,8 +102,16 @@ int main(int argc, char *argv[])
{
queryFile = argv[3];
}
std::cout << "Testing query file ..." << queryFile << std::endl;
if (rank == 0)
{
std::cout << " using config file = " << configFileName << std::endl;
std::cout << " data file = " << dataFileName << std::endl;
std::cout << " queryfile = " << queryFile << std::endl;
}

queryIDs(queryIO, dataFileName, queryFile);

std::cout << "\n" << std::endl;
queryWithStreaming(queryIO, dataFileName, queryFile);

return 0;
Expand All @@ -99,41 +126,3 @@ int main(int argc, char *argv[])

return 0;
}

bool testMe(std::string &queryConfigFile, std::string const &doubleVarName, MPI_Comm comm)
{
adios2::ADIOS ad(queryConfigFile, comm);
std::string dataFileName = "test.file";

// adios2::query::Worker w(queryConfigFile, comm);

// w.SetSource(inIO, reader);

{
// the config file should have info on idx method to use

/*
// if wanting to build customized minmax idx
// instead of using the existing block stats or bp4 minmax arrays
bool overwrite = false;
size_t recommendedSize = 20000;

if (!w.PrepareIdx(overwrite, recommendedSize, doubleVarName))
return false;
*/
}

adios2::IO inIO = ad.DeclareIO("query");
adios2::Engine reader = inIO.Open(dataFileName, adios2::Mode::Read, comm);

// to be continued
if (!reader)
return false;
// std::vector<double> dataOutput;
// std::vector<adios2::Dims> coordinateOutput;

return true;
}

/*
*/
134 changes: 81 additions & 53 deletions source/adios2/toolkit/query/BlockIndex.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,6 @@ namespace query
template <class T>
class BlockIndex
{
struct Tree
{
//
// ** no need to keep the original block. might be smaller than
// blockIndex typename Variable<T>::BPInfo& m_BlockInfo;
//
std::vector<typename adios2::core::Variable<T>::BPInfo> m_SubBlockInfo;
};

public:
BlockIndex<T>(adios2::core::Variable<T> *var, adios2::core::IO &io,
adios2::core::Engine &reader)
Expand All @@ -30,15 +21,20 @@ class BlockIndex

void Generate(std::string &fromBPFile, const adios2::Params &inputs) {}

void Evaluate(const QueryVar &query, std::vector<adios2::Box<adios2::Dims>> &resultSubBlocks)
void Evaluate(const QueryVar &query, std::vector<BlockHit> &resultBlockIDs)
{
if (nullptr == m_VarPtr)
{
throw std::runtime_error("Unable to evaluate query! Invalid Variable detected");
}

if (m_IdxReader.m_EngineType.find("5") != std::string::npos) // a bp5 reader
RunBP5Stat(query, resultSubBlocks);
RunBP5Stat(query, resultBlockIDs);
else
RunBP4Stat(query, resultSubBlocks);
RunBP4Stat(query, resultBlockIDs);
}

void RunBP5Stat(const QueryVar &query, std::vector<adios2::Box<adios2::Dims>> &hitBlocks)
void RunBP5Stat(const QueryVar &query, std::vector<BlockHit> &hitBlocks)
{
size_t currStep = m_IdxReader.CurrentStep();
adios2::Dims currShape = m_VarPtr->Shape();
Expand All @@ -52,29 +48,40 @@ class BlockIndex
}
for (auto &blockInfo : MinBlocksInfo->BlocksInfo)
{
Dims ss(MinBlocksInfo->Dims);
Dims cc(MinBlocksInfo->Dims);
for (std::vector<int>::size_type i = 0; i < ss.size(); i++)
{
ss[i] = blockInfo.Start[i];
cc[i] = blockInfo.Count[i];
}
if (!query.TouchSelection(ss, cc))
continue;

T bmin = *(T *)&blockInfo.MinMax.MinUnion;
T bmax = *(T *)&blockInfo.MinMax.MaxUnion;
bool isHit = query.m_RangeTree.CheckInterval(bmin, bmax);
if (isHit)

if (!isHit)
continue;

if (m_VarPtr->m_ShapeID != adios2::ShapeID::LocalArray)
{
adios2::Box<adios2::Dims> box = {ss, cc};
hitBlocks.push_back(box);
Dims ss(MinBlocksInfo->Dims);
Dims cc(MinBlocksInfo->Dims);
for (std::vector<int>::size_type i = 0; i < ss.size(); i++)
{
ss[i] = blockInfo.Start[i];
cc[i] = blockInfo.Count[i];
}
if (!query.TouchSelection(ss, cc))
continue;

if (isHit)
{
adios2::Box<adios2::Dims> box = {ss, cc};
hitBlocks.push_back(BlockHit(blockInfo.BlockID, box));
}
}
else
{ // local array
hitBlocks.push_back(BlockHit(blockInfo.BlockID));
}
}
delete MinBlocksInfo;
}

void RunBP4Stat(const QueryVar &query, std::vector<adios2::Box<adios2::Dims>> &hitBlocks)
void RunBP4Stat(const QueryVar &query, std::vector<BlockHit> &hitBlocks)
{
size_t currStep = m_IdxReader.CurrentStep();
adios2::Dims currShape = m_VarPtr->Shape();
Expand All @@ -86,45 +93,66 @@ class BlockIndex

for (auto &blockInfo : varBlocksInfo)
{
if (!query.TouchSelection(blockInfo.Start, blockInfo.Count))
bool isHit = query.m_RangeTree.CheckInterval(blockInfo.Min, blockInfo.Max);
if (!isHit)
continue;

if (blockInfo.MinMaxs.size() > 0)
if (m_VarPtr->m_ShapeID == adios2::ShapeID::LocalArray)
{
adios2::helper::CalculateSubblockInfo(blockInfo.Count, blockInfo.SubBlockInfo);
unsigned int numSubBlocks = static_cast<unsigned int>(blockInfo.MinMaxs.size() / 2);
for (unsigned int i = 0; i < numSubBlocks; i++)
if (isHit)
hitBlocks.push_back(BlockHit(blockInfo.BlockID));
}
else
{
// global array
if (!query.TouchSelection(blockInfo.Start, blockInfo.Count))
continue;

BlockHit tmp(blockInfo.BlockID);
if (blockInfo.MinMaxs.size() > 0)
{
bool isHit = query.m_RangeTree.CheckInterval(blockInfo.MinMaxs[2 * i],
blockInfo.MinMaxs[2 * i + 1]);
if (isHit)
// Consolidate to whole block If all subblocks are hits, then return the whole
// block
bool allCovered = true;

adios2::helper::CalculateSubblockInfo(blockInfo.Count, blockInfo.SubBlockInfo);
unsigned int numSubBlocks =
static_cast<unsigned int>(blockInfo.MinMaxs.size() / 2);
for (unsigned int i = 0; i < numSubBlocks; i++)
{
adios2::Box<adios2::Dims> currSubBlock =
adios2::helper::GetSubBlock(blockInfo.Count, blockInfo.SubBlockInfo, i);
for (size_t d = 0; d < blockInfo.Count.size(); ++d)
bool isSubblockHit = query.m_RangeTree.CheckInterval(
blockInfo.MinMaxs[2 * i], blockInfo.MinMaxs[2 * i + 1]);
if (isSubblockHit)
{
adios2::Box<adios2::Dims> currSubBlock = adios2::helper::GetSubBlock(
blockInfo.Count, blockInfo.SubBlockInfo, i);
for (size_t d = 0; d < blockInfo.Count.size(); ++d)
currSubBlock.first[d] += blockInfo.Start[d];

if (!query.TouchSelection(currSubBlock.first, currSubBlock.second))
continue;
tmp.m_Regions.push_back(currSubBlock);
}
else
{
currSubBlock.first[d] += blockInfo.Start[d];
allCovered = false;
}
if (!query.TouchSelection(currSubBlock.first, currSubBlock.second))
continue;
hitBlocks.push_back(currSubBlock);
} // for num subblocks

if (!allCovered)
{
hitBlocks.push_back(tmp);
continue;
}
}
}
else
{ // default
bool isHit = query.m_RangeTree.CheckInterval(blockInfo.Min, blockInfo.Max);
if (isHit)
{
adios2::Box<adios2::Dims> box = {blockInfo.Start, blockInfo.Count};
hitBlocks.push_back(box);
}

// no subblock info or (allCovered = true)
adios2::Box<adios2::Dims> box = {blockInfo.Start, blockInfo.Count};
hitBlocks.push_back(BlockHit(blockInfo.BlockID, box));
}
}
}

Tree m_Content;

// can not be unique_ptr as it changes with bp5 through steps
// as BP5Deserializer::SetupForStep calls io.RemoveVariables()
// must use ptr as bp5 associates ptrs with blockinfo, see MinBlocksInfo() in bp5
Expand Down
Loading
Loading