From cc9a45085176147cdd30927852aa4cde0df47084 Mon Sep 17 00:00:00 2001 From: Lucas Esclapez <13371051+esclapez@users.noreply.github.com> Date: Fri, 5 Aug 2022 10:43:23 -0700 Subject: [PATCH] Additional vars and diagnostics to efield (#113) * Add species mass balance dmYdt = Sum( Flux(A) + Flux(D) + R). * Update doc. * Add species balance to FlameSheet regtest. * Add cell-averaged ions fluxes to pltfile and add domain flux integrals. * Fix the ABecCec laplacian for AMReX updates. --- .../LinOps/AMReX_MLABecCecLaplacian.cpp | 6 ++--- Source/Efield/LinOps/AMReX_MLCellABecCecLap.H | 5 +++-- .../Efield/LinOps/AMReX_MLCellABecCecLap.cpp | 8 ++++--- Source/Efield/PeleLMEF.H | 8 +++++++ Source/Efield/PeleLMEFUtils.cpp | 18 +++++++++++++++ Source/PeleLM.H | 4 ++-- Source/PeleLMAdvection.cpp | 16 ++++++++++++++ Source/PeleLMInit.cpp | 3 +++ Source/PeleLMPlot.cpp | 22 +++++++++++++++++++ Source/PeleLMRegrid.cpp | 9 ++++++++ Source/PeleLMSetup.cpp | 12 ++++++++++ Source/PeleLMTemporals.cpp | 20 +++++++++++++++++ 12 files changed, 121 insertions(+), 10 deletions(-) diff --git a/Source/Efield/LinOps/AMReX_MLABecCecLaplacian.cpp b/Source/Efield/LinOps/AMReX_MLABecCecLaplacian.cpp index e29145d4..85fdd4d1 100644 --- a/Source/Efield/LinOps/AMReX_MLABecCecLaplacian.cpp +++ b/Source/Efield/LinOps/AMReX_MLABecCecLaplacian.cpp @@ -317,11 +317,11 @@ MLABecCecLaplacian::applyMetricTermsCoeffs () for (int alev = 0; alev < m_num_amr_levels; ++alev) { const int mglev = 0; - applyMetricTerm(alev, mglev, m_a_coeffs[alev][mglev]); + applyMetricTermToMF(alev, mglev, m_a_coeffs[alev][mglev]); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - applyMetricTerm(alev, mglev, m_b_coeffs[alev][mglev][idim]); - applyMetricTerm(alev, mglev, m_c_coeffs[alev][mglev][idim]); + applyMetricTermToMF(alev, mglev, m_b_coeffs[alev][mglev][idim]); + applyMetricTermToMF(alev, mglev, m_c_coeffs[alev][mglev][idim]); } } #endif diff --git a/Source/Efield/LinOps/AMReX_MLCellABecCecLap.H b/Source/Efield/LinOps/AMReX_MLCellABecCecLap.H index 6390c377..ae3d8eac 100644 --- a/Source/Efield/LinOps/AMReX_MLCellABecCecLap.H +++ b/Source/Efield/LinOps/AMReX_MLCellABecCecLap.H @@ -1,5 +1,6 @@ #ifndef AMREX_ML_CELL_ABECCECLAP_H_ #define AMREX_ML_CELL_ABECCECLAP_H_ +#include #include @@ -60,9 +61,9 @@ public: virtual Array getBCoeffs (int amrlev, int mglev) const = 0; virtual Array getCCoeffs (int amrlev, int mglev) const = 0; - virtual void applyInhomogNeumannTerm (int amrlev, MultiFab& rhs) const final override; + virtual void applyInhomogNeumannTerm (int amrlev, Any& a_rhs) const final override; - virtual void applyOverset (int amlev, MultiFab& rhs) const override; + virtual void applyOverset (int amlev, Any& a_rhs) const override; #ifdef AMREX_USE_HYPRE virtual std::unique_ptr makeHypre (Hypre::Interface hypre_interface) const override; #endif diff --git a/Source/Efield/LinOps/AMReX_MLCellABecCecLap.cpp b/Source/Efield/LinOps/AMReX_MLCellABecCecLap.cpp index fa581089..755bf416 100644 --- a/Source/Efield/LinOps/AMReX_MLCellABecCecLap.cpp +++ b/Source/Efield/LinOps/AMReX_MLCellABecCecLap.cpp @@ -108,7 +108,7 @@ MLCellABecCecLap::define (const Vector& a_geom, amrlev = 0; for (int mglev = 1; mglev < m_num_mg_levels[amrlev]; ++mglev) { MultiFab foo(m_grids[amrlev][mglev], m_dmap[amrlev][mglev], 1, 0, MFInfo().SetAlloc(false)); - if (! isMFIterSafe(*m_overset_mask[amrlev][mglev], foo)) { + if (! amrex::isMFIterSafe(*m_overset_mask[amrlev][mglev], foo)) { auto osm = std::make_unique(m_grids[amrlev][mglev], m_dmap[amrlev][mglev], 1, 1); osm->ParallelCopy(*m_overset_mask[amrlev][mglev]); @@ -194,7 +194,7 @@ MLCellABecCecLap::getFluxes (const Vector >& a_f } void -MLCellABecCecLap::applyInhomogNeumannTerm (int amrlev, MultiFab& rhs) const +MLCellABecCecLap::applyInhomogNeumannTerm (int amrlev, Any& a_rhs) const { bool has_inhomog_neumann = hasInhomogNeumannBC(); bool has_robin = hasRobinBC(); @@ -206,9 +206,11 @@ MLCellABecCecLap::applyInhomogNeumannTerm (int amrlev, MultiFab& rhs) const } void -MLCellABecCecLap::applyOverset (int amrlev, MultiFab& rhs) const +MLCellABecCecLap::applyOverset (int amrlev, Any& a_rhs) const { if (m_overset_mask[amrlev][0]) { + AMREX_ASSERT(a_rhs.is()); + auto& rhs = a_rhs.get(); const int ncomp = getNComp(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) diff --git a/Source/Efield/PeleLMEF.H b/Source/Efield/PeleLMEF.H index 78a1074c..ca87a4c1 100644 --- a/Source/Efield/PeleLMEF.H +++ b/Source/Efield/PeleLMEF.H @@ -68,3 +68,11 @@ amrex::Vector getNLresidVect(); amrex::Vector getNLstateVect(); amrex::Vector getNLBGChargeVect(); + + // Temporals + int m_do_ionsBalance = 0; + std::ofstream tmpIonsFile; + + // Extra diagnostics + int m_do_extraEFdiags = 0; + amrex::Vector> m_ionsFluxes; diff --git a/Source/Efield/PeleLMEFUtils.cpp b/Source/Efield/PeleLMEFUtils.cpp index 68e834bd..0b282290 100644 --- a/Source/Efield/PeleLMEFUtils.cpp +++ b/Source/Efield/PeleLMEFUtils.cpp @@ -329,3 +329,21 @@ void PeleLM::fillPatchNLphiV(Real a_time, refRatio(lev-1), mapper, fetchBCRecArray(PHIV,1), 0); } } + +void PeleLM::ionsBalance() +{ + // Compute the sum of ions on the domain boundaries + Array ionsCurrent{0.0}; + for (int n = NUM_SPECIES-NUM_IONS; n < NUM_SPECIES; n++){ + for (int i = 0; i < 2*AMREX_SPACEDIM; i++) { + ionsCurrent[i] += m_domainRhoYFlux[2*n*AMREX_SPACEDIM+i] * zk[n]; + } + } + + tmpIonsFile << m_nstep << " " << m_cur_time; // Time info + for (int i = 0; i < 2*AMREX_SPACEDIM; i++) { + tmpIonsFile << " " << ionsCurrent[i]; // ions current as xlo, xhi, ylo, ... + } + tmpIonsFile << "\n"; + tmpIonsFile.flush(); +} diff --git a/Source/PeleLM.H b/Source/PeleLM.H index 1f09e79c..cacbffc8 100644 --- a/Source/PeleLM.H +++ b/Source/PeleLM.H @@ -905,12 +905,12 @@ class PeleLM : public amrex::AmrCore { void jTimesV(const amrex::Vector &a_x, const amrex::Vector &a_Ax); - - void setUpPrecond(const amrex::Real &a_dt, const amrex::Vector &a_nE); void applyPrecond(const amrex::Vector &a_v, const amrex::Vector &a_Pv); + + void ionsBalance(); //----------------------------------------------------------------------------- #endif diff --git a/Source/PeleLMAdvection.cpp b/Source/PeleLMAdvection.cpp index 656a62b5..8d16f037 100644 --- a/Source/PeleLMAdvection.cpp +++ b/Source/PeleLMAdvection.cpp @@ -639,6 +639,22 @@ void PeleLM::computeScalarAdvTerms(std::unique_ptr &advData) // Compute face domain integral for U at every SDC iteration addUmacFluxes(advData, geom[0]); +#ifdef PELE_USE_EFIELD + if (m_do_extraEFdiags) { + for (int lev = 0; lev <= finest_level; ++lev) { + for (int n = 0; n < NUM_IONS; ++n) { + int spec_idx = NUM_SPECIES - NUM_IONS + n; + Array,AMREX_SPACEDIM> ionFlux; + for (int idim = 0; idim < AMREX_SPACEDIM; idim++ ) { + ionFlux[idim].reset(new MultiFab(fluxes[lev][idim],amrex::make_alias,spec_idx,1)); + } + average_face_to_cellcenter(*m_ionsFluxes[lev],n*AMREX_SPACEDIM, + GetArrOfConstPtrs(ionFlux)); + } + } + } +#endif + //---------------------------------------------------------------- // Fluxes divergence to get the scalars advection term auto AdvTypeAll = fetchAdvTypeArray(FIRSTSPEC,NUM_SPECIES+1); // Species+RhoH diff --git a/Source/PeleLMInit.cpp b/Source/PeleLMInit.cpp index 75ab035e..fb700415 100644 --- a/Source/PeleLMInit.cpp +++ b/Source/PeleLMInit.cpp @@ -70,6 +70,9 @@ void PeleLM::MakeNewLevelFromScratch( int lev, #ifdef PELE_USE_EFIELD m_leveldatanlsolve[lev].reset(new LevelDataNLSolve(grids[lev], dmap[lev], *m_factory[lev], m_nGrowState)); + if (m_do_extraEFdiags) { + m_ionsFluxes[lev].reset(new MultiFab(grids[lev], dmap[lev], NUM_IONS*AMREX_SPACEDIM, 0)); + } #endif // Fill the initial solution (if not restarting) diff --git a/Source/PeleLMPlot.cpp b/Source/PeleLMPlot.cpp index 187ade95..bd4c060b 100644 --- a/Source/PeleLMPlot.cpp +++ b/Source/PeleLMPlot.cpp @@ -99,6 +99,12 @@ void PeleLM::WritePlotFile() { } #endif +#ifdef PELE_USE_EFIELD + if (m_do_extraEFdiags) { + ncomp += NUM_IONS * AMREX_SPACEDIM; + } +#endif + //---------------------------------------------------------------- // Plot MultiFabs Vector mf_plt(finest_level + 1); @@ -187,6 +193,17 @@ void PeleLM::WritePlotFile() { } #endif +#ifdef PELE_USE_EFIELD + if (m_do_extraEFdiags) { + for (int ivar = 0; ivar < NUM_IONS; ++ivar) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + std::string dir = (idim == 0) ? "X" : ( (idim == 1) ? "Y" : "Z"); + plt_VarsName.push_back("DriftFlux_"+names[NUM_SPECIES-NUM_IONS+ivar]+"_"+dir); + } + } + } +#endif + //---------------------------------------------------------------- // Fill the plot MultiFabs for (int lev = 0; lev <= finest_level; ++lev) { @@ -266,6 +283,11 @@ void PeleLM::WritePlotFile() { cnt += num_spray_derive; } #endif +#ifdef PELE_USE_EFIELD + if (m_do_extraEFdiags) { + MultiFab::Copy(mf_plt[lev], *m_ionsFluxes[lev], 0, cnt, m_ionsFluxes[lev]->nComp(),0); + } +#endif #ifdef AMREX_USE_EB EB_set_covered(mf_plt[lev],0.0); #endif diff --git a/Source/PeleLMRegrid.cpp b/Source/PeleLMRegrid.cpp index be01fe5c..9d80a1d1 100644 --- a/Source/PeleLMRegrid.cpp +++ b/Source/PeleLMRegrid.cpp @@ -86,6 +86,9 @@ void PeleLM::MakeNewLevelFromCoarse( int lev, #ifdef PELE_USE_EFIELD m_leveldatanlsolve[lev].reset(new LevelDataNLSolve(ba, dm, *m_factory[lev], m_nGrowState)); + if (m_do_extraEFdiags) { + m_ionsFluxes[lev].reset(new MultiFab(ba, dm, NUM_IONS*AMREX_SPACEDIM, 0)); + } m_precond_op.reset(); #endif @@ -173,6 +176,9 @@ void PeleLM::RemakeLevel( int lev, #ifdef PELE_USE_EFIELD m_leveldatanlsolve[lev].reset(new LevelDataNLSolve(ba, dm, *m_factory[lev], m_nGrowState)); + if (m_do_extraEFdiags) { + m_ionsFluxes[lev].reset(new MultiFab(ba, dm, NUM_IONS*AMREX_SPACEDIM, 0)); + } m_precond_op.reset(); #endif @@ -204,6 +210,9 @@ void PeleLM::ClearLevel(int lev) { macproj.reset(); #ifdef PELE_USE_EFIELD m_leveldatanlsolve[lev].reset(); + if (m_do_extraEFdiags) { + m_ionsFluxes[lev].reset(); + } #endif m_extSource[lev]->clear(); } diff --git a/Source/PeleLMSetup.cpp b/Source/PeleLMSetup.cpp index 0cf048d8..9e8239a7 100644 --- a/Source/PeleLMSetup.cpp +++ b/Source/PeleLMSetup.cpp @@ -464,7 +464,18 @@ void PeleLM::readParameters() { ppef.query("restart_nonEF",m_restart_nonEF); ppef.query("restart_electroneutral",m_restart_electroneutral); ppef.query("restart_resetTime",m_restart_resetTime); + ppef.query("plot_extras",m_do_extraEFdiags); + + // Getting the ions fluxes on the domain boundaries + // Species balance data is needed, so override if not activated + if (m_do_temporals) { + ppef.query("do_ionsBalance",m_do_ionsBalance); + if (m_do_ionsBalance) { + m_do_speciesBalance = 1; + } + } #endif + #ifdef PELELM_USE_SPRAY readSprayParameters(); #endif @@ -947,6 +958,7 @@ void PeleLM::resizeArray() { #ifdef PELE_USE_EFIELD m_leveldatanlsolve.resize(max_level+1); + m_ionsFluxes.resize(max_level+1); #endif // External sources diff --git a/Source/PeleLMTemporals.cpp b/Source/PeleLMTemporals.cpp index cb21b134..dfff77f9 100644 --- a/Source/PeleLMTemporals.cpp +++ b/Source/PeleLMTemporals.cpp @@ -425,6 +425,13 @@ void PeleLM::writeTemporals() tmpExtremasFile << " \n"; tmpExtremasFile.flush(); + +#ifdef PELE_USE_EFIELD + if (m_do_ionsBalance) { + ionsBalance(); + } +#endif + } void PeleLM::openTempFile() @@ -453,6 +460,13 @@ void PeleLM::openTempFile() tmpExtremasFile.open(tempFileName.c_str(),std::ios::out | std::ios::app | std::ios_base::binary); tmpExtremasFile.precision(12); } +#ifdef PELE_USE_EFIELD + if (m_do_ionsBalance) { + tempFileName = "temporals/tempIons"; + tmpIonsFile.open(tempFileName.c_str(),std::ios::out | std::ios::app | std::ios_base::binary); + tmpIonsFile.precision(12); + } +#endif } } @@ -475,5 +489,11 @@ void PeleLM::closeTempFile() tmpExtremasFile.flush(); tmpExtremasFile.close(); } +#ifdef PELE_USE_EFIELD + if (m_do_ionsBalance) { + tmpIonsFile.flush(); + tmpIonsFile.close(); + } +#endif } }