From aedc392c37ad010343864b1ba55b60af18b7cc32 Mon Sep 17 00:00:00 2001 From: Lucas Esclapez Date: Fri, 24 Mar 2023 18:17:17 -0700 Subject: [PATCH 1/5] Need a ghost cell when processing the diagnostics. --- Source/PeleLMDiagnostics.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/PeleLMDiagnostics.cpp b/Source/PeleLMDiagnostics.cpp index 1e51fa0c..6804d9c5 100644 --- a/Source/PeleLMDiagnostics.cpp +++ b/Source/PeleLMDiagnostics.cpp @@ -57,16 +57,16 @@ PeleLM::doDiagnostics() // Assemble a vector of MF containing the requested data Vector > diagMFVec(finestLevel()+1); for (int lev{0}; lev <= finestLevel(); ++lev) { - diagMFVec[lev] = std::make_unique(grids[lev], dmap[lev], m_diagVars.size(), 0); + diagMFVec[lev] = std::make_unique(grids[lev], dmap[lev], m_diagVars.size(), 1); for (int v{0}; v < m_diagVars.size(); ++v ) { std::unique_ptr mf; - mf = derive(m_diagVars[v], m_cur_time, lev, 0); + mf = derive(m_diagVars[v], m_cur_time, lev, 1); // If the variable is a derive component, get its index from the derive multifab // TODO: if multiple diagVars are components of the same derive, they get redundantly derived each time int mf_idx = 0; const PeleLMDeriveRec* rec = derive_lst.get(m_diagVars[v]); if (rec) mf_idx = rec->variableComp(m_diagVars[v]); - MultiFab::Copy(*diagMFVec[lev].get(), *mf, mf_idx, v, 1, 0); + MultiFab::Copy(*diagMFVec[lev].get(), *mf, mf_idx, v, 1, 1); } } From 77ce8c4ed1e5843a4167ad7d6a91080ba4b6be0e Mon Sep 17 00:00:00 2001 From: Lucas Esclapez Date: Fri, 24 Mar 2023 18:18:22 -0700 Subject: [PATCH 2/5] Directly handle 2D in here instead of re-writting after 3D AMReX call. --- Source/Diagnostics/DiagFramePlane.H | 18 +- Source/Diagnostics/DiagFramePlane.cpp | 359 ++++++++++++++------------ 2 files changed, 201 insertions(+), 176 deletions(-) diff --git a/Source/Diagnostics/DiagFramePlane.H b/Source/Diagnostics/DiagFramePlane.H index 454f9fe6..42ccf3b7 100644 --- a/Source/Diagnostics/DiagFramePlane.H +++ b/Source/Diagnostics/DiagFramePlane.H @@ -2,6 +2,7 @@ #define _DIAGFRAMEPLANE_H_ #include "DiagBase.H" +#include class DiagFramePlane : public DiagBase::Register { @@ -38,13 +39,22 @@ public: void VisMF2D(const amrex::MultiFab &a_mf, const std::string& a_mf_name); + void Write2DMFHeader(const std::string &a_mf_name, + amrex::VisMF::Header &hdr, + int coordinatorProc, + MPI_Comm comm); + + void Find2FOffsets(const amrex::FabArray &mf, + const std::string &filePrefix, + amrex::VisMF::Header &hdr, + amrex::VisMF::Header::Version /*whichVersion*/, + amrex::NFilesIter &nfi, + int nOutFile, MPI_Comm comm); + void write_2D_header(std::ostream &os, const amrex::FArrayBox& f, int nvar); - int diff_2D3D_header(const amrex::Box a_box, - int nvar); - void Write2DPlotfileHeader(std::ostream &HeaderFile, int nlevels, const amrex::Vector &bArray, @@ -57,8 +67,6 @@ public: const std::string &levelPrefix = "Level_", const std::string &mfPrefix = "Cell"); - void ReWriteLevelVisMFHeader(const std::string &a_HeaderPath); - void close() override {} private: diff --git a/Source/Diagnostics/DiagFramePlane.cpp b/Source/Diagnostics/DiagFramePlane.cpp index e35489d7..3c138f42 100644 --- a/Source/Diagnostics/DiagFramePlane.cpp +++ b/Source/Diagnostics/DiagFramePlane.cpp @@ -301,6 +301,8 @@ DiagFramePlane::Write2DMultiLevelPlotfile(const std::string &a_pltfile, amrex::ParallelDescriptor::Barrier(); if (amrex::ParallelDescriptor::MyProc() == amrex::ParallelDescriptor::NProcs()-1) { + + // Write 2D pltfile header amrex::Vector boxArrays(a_nlevels); for(int level(0); level < boxArrays.size(); ++level) { boxArrays[level] = a_slice[level]->boxArray(); @@ -317,12 +319,32 @@ DiagFramePlane::Write2DMultiLevelPlotfile(const std::string &a_pltfile, Write2DPlotfileHeader(HeaderFile, a_nlevels, boxArrays, a_varnames, a_geoms, a_time, a_steps, a_rref, versionName, levelPrefix, mfPrefix); + HeaderFile.flush(); + HeaderFile.close(); + + // Add extra file providing the plane information + // because the output is always a z-normal plane at z=0 + std::string PlaneFileName(a_pltfile + "/PlaneData"); + std::ofstream PlaneFile; + PlaneFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); + PlaneFile.open(PlaneFileName.c_str(), std::ofstream::out | + std::ofstream::trunc | + std::ofstream::binary); + if( ! PlaneFile.good()) amrex::FileOpenFailed(PlaneFileName); + + PlaneFile << "Pele plane definition: \n"; + PlaneFile << m_normal << "\n"; + PlaneFile << AMREX_D_TERM(m_center[0], + << " " << m_center[1], + << " " << m_center[2]) << "\n"; + + PlaneFile.flush(); + PlaneFile.close(); } + + // Write a 2D version of the MF at each level for (int level = 0; level < a_nlevels; ++level) { - VisMF2D(*a_slice[level], amrex::MultiFabFileFullPrefix(level, a_pltfile, levelPrefix, mfPrefix+"2D")); - amrex::VisMF::Write(*a_slice[level], amrex::MultiFabFileFullPrefix(level, a_pltfile, levelPrefix, mfPrefix)); - amrex::ParallelDescriptor::Barrier(); - ReWriteLevelVisMFHeader(amrex::MultiFabFileFullPrefix(level, a_pltfile, levelPrefix, "")); + VisMF2D(*a_slice[level], amrex::MultiFabFileFullPrefix(level, a_pltfile, levelPrefix, mfPrefix)); } } @@ -402,147 +424,6 @@ DiagFramePlane::Write2DPlotfileHeader(std::ostream &HeaderFile, } } -void -DiagFramePlane::ReWriteLevelVisMFHeader(const std::string &a_HeaderPath) -{ - std::string OldHeaderFileName(a_HeaderPath+ "Cell_H"); - amrex::Vector oldfileCharPtr; - amrex::ParallelDescriptor::ReadAndBcastFile(OldHeaderFileName, oldfileCharPtr); - - if (amrex::ParallelDescriptor::IOProcessor()) { - amrex::VisMF::IO_Buffer io_buffer_new(amrex::VisMF::IO_Buffer_Size); - std::string HeaderFileName(a_HeaderPath+ "Cell_temp_H"); - std::ofstream HeaderFile; - HeaderFile.rdbuf()->pubsetbuf(io_buffer_new.dataPtr(), io_buffer_new.size()); - HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | - std::ofstream::trunc | - std::ofstream::binary); - if( ! HeaderFile.good()) amrex::FileOpenFailed(HeaderFileName); - - std::string fileCharPtrString(oldfileCharPtr.dataPtr()); - std::istringstream is(fileCharPtrString, std::istringstream::in); - std::string line, word; - - // Version, How, nComp, nGrow - int version, how, nComp, nGrow; - is >> version; - is.ignore(std::streamsize(10000), '\n'); - is >> how; - is.ignore(std::streamsize(10000), '\n'); - is >> nComp; - is.ignore(std::streamsize(10000), '\n'); - is >> nGrow; - - HeaderFile << version << "\n"; - HeaderFile << how << "\n"; - HeaderFile << nComp << "\n"; - HeaderFile << nGrow << "\n"; - - // BoxArray - int nbox = -1, dummy = 0; - is.ignore(std::streamsize(10000), '(') >> nbox >> dummy; - is.ignore(std::streamsize(10000), '\n'); - HeaderFile << '(' << nbox << ' ' << 0 << '\n'; - amrex::Vector fullDimBox(nbox); - for (int i = 0; i < nbox; ++i) { - is >> fullDimBox[i]; - printLowerDimBox(HeaderFile, fullDimBox[i], 2); - HeaderFile << '\n'; - } - is.ignore(std::streamsize(10000), '\n'); - std::getline(is, line); - HeaderFile << line << "\n"; - - //------------------------------------------------------------------ - // Get list of Datafiles and correct for FAB header length in offset - //------------------------------------------------------------------ - std::getline(is, line); - HeaderFile << line << "\n"; - int nFabs = std::stoi(line); - - // Stash away lines and count the number of files - amrex::Vector dataFiles; - amrex::Vector fablines(nFabs); - for (int i = 0; i < nFabs; ++i) { - std::getline(is, fablines[i]); - std::istringstream lis(fablines[i]); - std::string w1, w2, w3; - lis >> w1; - lis >> w2; - lis >> w3; - int offset = std::stoi(w3); - if (offset==0) dataFiles.push_back(w2); - } - - // Map files and FABs - int nDataFiles = dataFiles.size(); - amrex::Vector nFabsInFiles(nDataFiles); - amrex::Vector > FabsInFiles(nDataFiles); - for (int i = 0; i < nFabs; ++i) { - std::istringstream lis(fablines[i]); - std::string w1, w2, w3; - lis >> w1; - lis >> w2; - lis >> w3; - for (int f = 0; f < nDataFiles; ++f) { - if (w2 == dataFiles[f]) { - nFabsInFiles[f]+=1; - FabsInFiles[f].push_back(i); - break; - } - } - } - // Check - for (int f = 0; f < nDataFiles; ++f) { - if (nFabsInFiles[f] != FabsInFiles[f].size()) { - amrex::Abort("Something went wrong while counting FABs in files"); - } - } - - // Write out the FAB on disk list, corrected - for (int i = 0; i < nFabs; ++i) { - std::istringstream lis(fablines[i]); - std::string w1, w2, w3; - lis >> w1; - lis >> w2; - lis >> w3; - int offset = std::stoi(w3); - for (int f = 0; f < nDataFiles; ++f) { - if (w2 == dataFiles[f]) { - for (int fa = 0; fa < FabsInFiles[f].size(); ++fa) { - if (FabsInFiles[f][fa] == i) { - if (fa > 0) { - for (int fback = fa-1; fback >= 0; --fback) { - offset -= diff_2D3D_header(fullDimBox[fback],nComp); - } - } - } - } - } - } - HeaderFile << w1 << " " << w2 << " " << offset << "\n"; - } - - // Just pass from istream to ostream `till the end - while (is) { - std::getline(is, line); - HeaderFile << line << "\n"; - } - HeaderFile.close(); - - // Replace header file - std::rename(HeaderFileName.c_str(), OldHeaderFileName.c_str()); - } - - // Replace 3D data file by 2D ones - std::string oldname = amrex::Concatenate(a_HeaderPath+"Cell2D_D_",amrex::ParallelDescriptor::MyProc(),5); - if (amrex::FileSystem::Exists(oldname.c_str())) { - std::string newname = std::regex_replace(oldname, std::regex("Cell2D_"), "Cell_"); - std::remove(newname.c_str()); - std::rename(oldname.c_str(), newname.c_str()); - } -} - void DiagFramePlane::VisMF2D(const amrex::MultiFab& a_mf, const std::string& a_mf_name) @@ -667,6 +548,168 @@ DiagFramePlane::VisMF2D(const amrex::MultiFab& a_mf, } } } + + int coordinatorProc(amrex::ParallelDescriptor::IOProcessorNumber()); + if(nfi.GetDynamic()) { + coordinatorProc = nfi.CoordinatorProc(); + } + hdr.CalculateMinMax(a_mf, coordinatorProc); + + Find2FOffsets(a_mf, filePrefix, hdr, currentVersion, nfi, + nOutFiles, amrex::ParallelDescriptor::Communicator()); + + Write2DMFHeader(a_mf_name, hdr, coordinatorProc, amrex::ParallelDescriptor::Communicator()); +} + +void +DiagFramePlane::Write2DMFHeader(const std::string &a_mf_name, + amrex::VisMF::Header &hdr, + int coordinatorProc, + MPI_Comm comm) +{ + const int myProc(amrex::ParallelDescriptor::MyProc(comm)); + if(myProc == coordinatorProc) { + + std::string MFHdrFileName(a_mf_name); + + MFHdrFileName += "_H"; + + amrex::VisMF::IO_Buffer io_buffer(amrex::VisMF::IO_Buffer_Size); + + std::ofstream MFHdrFile; + + MFHdrFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); + MFHdrFile.open(MFHdrFileName.c_str(), std::ios::out | std::ios::trunc); + + if( ! MFHdrFile.good()) { + amrex::FileOpenFailed(MFHdrFileName); + } + + std::ios::fmtflags oflags = MFHdrFile.flags(); + MFHdrFile.setf(std::ios::floatfield, std::ios::scientific); + MFHdrFile << hdr.m_vers << '\n'; + MFHdrFile << int(hdr.m_how) << '\n'; + MFHdrFile << hdr.m_ncomp << '\n'; + if (hdr.m_ngrow == hdr.m_ngrow[0]) { + MFHdrFile << hdr.m_ngrow[0] << '\n'; + } else { + MFHdrFile << hdr.m_ngrow << '\n'; + } + + MFHdrFile << "(" << hdr.m_ba.size() << " 0" <<'\n'; + for (int i = 0; i < hdr.m_ba.size(); ++i) { + printLowerDimBox(MFHdrFile, hdr.m_ba[i], 2); + MFHdrFile << '\n'; + } + MFHdrFile << ") \n"; + + MFHdrFile << hdr.m_fod << '\n'; + + MFHdrFile << hdr.m_min.size() << "," << hdr.m_min[0].size() << '\n'; + MFHdrFile.precision(16); + for (int i = 0; i < hdr.m_min.size(); ++i) { + for (int j = 0; j < hdr.m_min[i].size(); ++j) { + MFHdrFile << hdr.m_min[i][j] << ","; + } + MFHdrFile << "\n"; + } + + MFHdrFile << "\n"; + + MFHdrFile << hdr.m_max.size() << "," << hdr.m_max[0].size() << '\n'; + for (int i = 0; i < hdr.m_max.size(); ++i) { + for (int j = 0; j < hdr.m_max[i].size(); ++j) { + MFHdrFile << hdr.m_max[i][j] << ","; + } + MFHdrFile << "\n"; + } + + MFHdrFile.flush(); + MFHdrFile.close(); + } +} + +void +DiagFramePlane::Find2FOffsets(const amrex::FabArray &mf, + const std::string &filePrefix, + amrex::VisMF::Header &hdr, + amrex::VisMF::Header::Version /*whichVersion*/, + amrex::NFilesIter &nfi, int nOutFiles, MPI_Comm comm) +{ + bool groupSets = false; + + const int myProc(amrex::ParallelDescriptor::MyProc(comm)); + const int nProcs(amrex::ParallelDescriptor::NProcs(comm)); + int coordinatorProc(amrex::ParallelDescriptor::IOProcessorNumber(comm)); + if(nfi.GetDynamic()) { + coordinatorProc = nfi.CoordinatorProc(); + } + + auto whichRD = amrex::FArrayBox::getDataDescriptor(); + const amrex::FABio &fio = amrex::FArrayBox::getFABio(); + int whichRDBytes(whichRD->numBytes()); + int nComps(mf.nComp()); + + if(myProc == coordinatorProc) { // ---- calculate offsets + const amrex::BoxArray &mfBA = mf.boxArray(); + const amrex::DistributionMapping &mfDM = mf.DistributionMap(); + amrex::Vector fabHeaderBytes(mfBA.size(), 0); + int nFiles(amrex::NFilesIter::ActualNFiles(nOutFiles)); + int whichFileNumber(-1); + std::string whichFileName; + amrex::Vector currentOffset(nProcs, 0L); + + // ---- find the length of the fab header instead of asking the file system + for(int i(0); i < mfBA.size(); ++i) { + std::stringstream hss; + amrex::FArrayBox tempFab(mf.fabbox(i), nComps, false); // ---- no alloc + write_2D_header(hss, tempFab, tempFab.nComp()); + fabHeaderBytes[i] = static_cast(hss.tellp()); + } + + std::map > rankBoxOrder; // ---- [rank, boxarray index array] + for(int i(0); i < mfBA.size(); ++i) { + rankBoxOrder[mfDM[i]].push_back(i); + } + + amrex::Vector fileNumbers; + if(nfi.GetDynamic()) { + fileNumbers = nfi.FileNumbersWritten(); + } + else if(nfi.GetSparseFPP()) { // if sparse, write to (file number = rank) + fileNumbers.resize(nProcs); + for(int i(0); i < nProcs; ++i) { + fileNumbers[i] = i; + } + } + else { + fileNumbers.resize(nProcs); + for(int i(0); i < nProcs; ++i) { + fileNumbers[i] = amrex::NFilesIter::FileNumber(nFiles, i, groupSets); + } + } + const amrex::Vector< amrex::Vector > &fileNumbersWriteOrder = nfi.FileNumbersWriteOrder(); + + for(int fn(0); fn < fileNumbersWriteOrder.size(); ++fn) { + for(int ri(0); ri < fileNumbersWriteOrder[fn].size(); ++ri) { + int rank(fileNumbersWriteOrder[fn][ri]); + auto rboIter = rankBoxOrder.find(rank); + + if(rboIter != rankBoxOrder.end()) { + amrex::Vector &index = rboIter->second; + whichFileNumber = fileNumbers[rank]; + whichFileName = amrex::VisMF::BaseName(amrex::NFilesIter::FileName(whichFileNumber, filePrefix)); + + for(int i(0); i < index.size(); ++i) { + hdr.m_fod[index[i]].m_name = whichFileName; + hdr.m_fod[index[i]].m_head = currentOffset[whichFileNumber]; + currentOffset[whichFileNumber] += mf.fabbox(index[i]).numPts() * nComps * whichRDBytes + + fabHeaderBytes[index[i]]; + } + } + } + } + } } void @@ -681,29 +724,3 @@ DiagFramePlane::write_2D_header(std::ostream& os, os << ' ' << nvar << '\n'; } } - -int -DiagFramePlane::diff_2D3D_header(const amrex::Box a_box, - int nvar) -{ - std::stringstream hss2D; - std::stringstream hss3D; - hss2D << "FAB " << amrex::FPC::NativeRealDescriptor(); - amrex::StreamRetry sr(hss2D, "FABio_write_header", 4); - while(sr.TryOutput()) { - printLowerDimBox(hss2D, a_box, 2); // Skipping z-dir - hss2D << ' ' << nvar << '\n'; - } - - hss3D << "FAB " << amrex::FPC::NativeRealDescriptor(); - amrex::StreamRetry sr3(hss3D, "FABio_write_header", 4); - while(sr3.TryOutput()) { - hss3D << a_box; - hss3D << ' ' << nvar << '\n'; - } - - int Length2D = static_cast(hss2D.tellp()); - int Length3D = static_cast(hss3D.tellp()); - - return Length3D-Length2D; -} From 3757986b6a5a8424718804a153b836ed7bdd34f1 Mon Sep 17 00:00:00 2001 From: Lucas Esclapez Date: Mon, 27 Mar 2023 16:24:21 -0700 Subject: [PATCH 3/5] Abort if a derived with more than 1 component is required in the Diagnostics. --- Source/PeleLMDiagnostics.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Source/PeleLMDiagnostics.cpp b/Source/PeleLMDiagnostics.cpp index 6804d9c5..bea7b672 100644 --- a/Source/PeleLMDiagnostics.cpp +++ b/Source/PeleLMDiagnostics.cpp @@ -32,6 +32,15 @@ PeleLM::createDiagnostics() bool itexists = derive_lst.canDerive(v) || isStateVariable(v) || isReactVariable(v); if (!itexists) { Abort("Field "+v+" is not available"); + } else { + if (derive_lst.canDerive(v)) { + const PeleLMDeriveRec* rec = derive_lst.get(v); + if (rec->numDerive() > 1) { + std::string errmsg = "Diagnostics can't handle derived with more than 1 component at the moment.\n"; + errmsg += "Add the desired components individually.\n"; + Abort(errmsg); + } + } } } } From 0b493661eb631b3b322deb643ef55f9dccfb11fc Mon Sep 17 00:00:00 2001 From: Lucas Esclapez Date: Mon, 27 Mar 2023 16:26:05 -0700 Subject: [PATCH 4/5] Remove unused. --- Source/Diagnostics/DiagFramePlane.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/Source/Diagnostics/DiagFramePlane.cpp b/Source/Diagnostics/DiagFramePlane.cpp index 3c138f42..3411cd31 100644 --- a/Source/Diagnostics/DiagFramePlane.cpp +++ b/Source/Diagnostics/DiagFramePlane.cpp @@ -585,7 +585,6 @@ DiagFramePlane::Write2DMFHeader(const std::string &a_mf_name, amrex::FileOpenFailed(MFHdrFileName); } - std::ios::fmtflags oflags = MFHdrFile.flags(); MFHdrFile.setf(std::ios::floatfield, std::ios::scientific); MFHdrFile << hdr.m_vers << '\n'; MFHdrFile << int(hdr.m_how) << '\n'; @@ -646,7 +645,6 @@ DiagFramePlane::Find2FOffsets(const amrex::FabArray &mf, } auto whichRD = amrex::FArrayBox::getDataDescriptor(); - const amrex::FABio &fio = amrex::FArrayBox::getFABio(); int whichRDBytes(whichRD->numBytes()); int nComps(mf.nComp()); From e207cc26dc31f7a5ece984695cf34fe50638c4ee Mon Sep 17 00:00:00 2001 From: Bruce Perry Date: Mon, 27 Mar 2023 18:22:38 -0600 Subject: [PATCH 5/5] allow single components of multicomponent derive --- Source/PeleLMDiagnostics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/PeleLMDiagnostics.cpp b/Source/PeleLMDiagnostics.cpp index bea7b672..7037c35b 100644 --- a/Source/PeleLMDiagnostics.cpp +++ b/Source/PeleLMDiagnostics.cpp @@ -35,7 +35,7 @@ PeleLM::createDiagnostics() } else { if (derive_lst.canDerive(v)) { const PeleLMDeriveRec* rec = derive_lst.get(v); - if (rec->numDerive() > 1) { + if (rec->variableComp(v) < 0) { std::string errmsg = "Diagnostics can't handle derived with more than 1 component at the moment.\n"; errmsg += "Add the desired components individually.\n"; Abort(errmsg);