Additional vars and diagnostics to efield (#113)

* Add species mass balance dmYdt = Sum( Flux(A) + Flux(D) + R). * Update doc. * Add species balance to FlameSheet regtest. * Add cell-averaged ions fluxes to pltfile and add domain flux integrals. * Fix the ABecCec laplacian for AMReX updates.
AMReX-Combustion · Aug 5, 2022 · cc9a450 · cc9a450
1 parent 0f22cca
commit cc9a450
Show file tree

Hide file tree

Showing 12 changed files with 121 additions and 10 deletions.
diff --git a/Source/Efield/LinOps/AMReX_MLABecCecLaplacian.cpp b/Source/Efield/LinOps/AMReX_MLABecCecLaplacian.cpp
@@ -317,11 +317,11 @@ MLABecCecLaplacian::applyMetricTermsCoeffs ()
     for (int alev = 0; alev < m_num_amr_levels; ++alev)
     {
         const int mglev = 0;
-        applyMetricTerm(alev, mglev, m_a_coeffs[alev][mglev]);
+        applyMetricTermToMF(alev, mglev, m_a_coeffs[alev][mglev]);
         for (int idim = 0; idim < AMREX_SPACEDIM; ++idim)
         {
-            applyMetricTerm(alev, mglev, m_b_coeffs[alev][mglev][idim]);
-            applyMetricTerm(alev, mglev, m_c_coeffs[alev][mglev][idim]);
+            applyMetricTermToMF(alev, mglev, m_b_coeffs[alev][mglev][idim]);
+            applyMetricTermToMF(alev, mglev, m_c_coeffs[alev][mglev][idim]);
         }
     }
 #endif

diff --git a/Source/Efield/LinOps/AMReX_MLCellABecCecLap.H b/Source/Efield/LinOps/AMReX_MLCellABecCecLap.H
@@ -1,5 +1,6 @@
 #ifndef AMREX_ML_CELL_ABECCECLAP_H_
 #define AMREX_ML_CELL_ABECCECLAP_H_
+#include <AMReX_Config.H>
 
 #include <AMReX_MLCellLinOp.H>
 
@@ -60,9 +61,9 @@ public:
     virtual Array<MultiFab const*,AMREX_SPACEDIM> getBCoeffs (int amrlev, int mglev) const = 0;
     virtual Array<MultiFab const*,AMREX_SPACEDIM> getCCoeffs (int amrlev, int mglev) const = 0;
 
-    virtual void applyInhomogNeumannTerm (int amrlev, MultiFab& rhs) const final override;
+    virtual void applyInhomogNeumannTerm (int amrlev, Any& a_rhs) const final override;
 
-    virtual void applyOverset (int amlev, MultiFab& rhs) const override;
+    virtual void applyOverset (int amlev, Any& a_rhs) const override;
 #ifdef AMREX_USE_HYPRE
     virtual std::unique_ptr<Hypre> makeHypre (Hypre::Interface hypre_interface) const override;
 #endif

diff --git a/Source/Efield/LinOps/AMReX_MLCellABecCecLap.cpp b/Source/Efield/LinOps/AMReX_MLCellABecCecLap.cpp
@@ -108,7 +108,7 @@ MLCellABecCecLap::define (const Vector<Geometry>& a_geom,
     amrlev = 0;
     for (int mglev = 1; mglev < m_num_mg_levels[amrlev]; ++mglev) {
         MultiFab foo(m_grids[amrlev][mglev], m_dmap[amrlev][mglev], 1, 0, MFInfo().SetAlloc(false));
-        if (! isMFIterSafe(*m_overset_mask[amrlev][mglev], foo)) {
+        if (! amrex::isMFIterSafe(*m_overset_mask[amrlev][mglev], foo)) {
             auto osm = std::make_unique<iMultiFab>(m_grids[amrlev][mglev],
                                                    m_dmap[amrlev][mglev], 1, 1);
             osm->ParallelCopy(*m_overset_mask[amrlev][mglev]);
@@ -194,7 +194,7 @@ MLCellABecCecLap::getFluxes (const Vector<Array<MultiFab*,AMREX_SPACEDIM> >& a_f
 }
 
 void
-MLCellABecCecLap::applyInhomogNeumannTerm (int amrlev, MultiFab& rhs) const
+MLCellABecCecLap::applyInhomogNeumannTerm (int amrlev, Any& a_rhs) const
 {
     bool has_inhomog_neumann = hasInhomogNeumannBC();
     bool has_robin = hasRobinBC();
@@ -206,9 +206,11 @@ MLCellABecCecLap::applyInhomogNeumannTerm (int amrlev, MultiFab& rhs) const
 }
 
 void
-MLCellABecCecLap::applyOverset (int amrlev, MultiFab& rhs) const
+MLCellABecCecLap::applyOverset (int amrlev, Any& a_rhs) const
 {
     if (m_overset_mask[amrlev][0]) {
+        AMREX_ASSERT(a_rhs.is<MultiFab>());
+        auto& rhs = a_rhs.get<MultiFab>();
         const int ncomp = getNComp();
 #ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())

diff --git a/Source/Efield/PeleLMEF.H b/Source/Efield/PeleLMEF.H
@@ -68,3 +68,11 @@
    amrex::Vector<amrex::MultiFab*> getNLresidVect();
    amrex::Vector<amrex::MultiFab*> getNLstateVect();
    amrex::Vector<amrex::MultiFab*> getNLBGChargeVect();
+
+   // Temporals
+   int m_do_ionsBalance = 0;
+   std::ofstream tmpIonsFile;
+
+   // Extra diagnostics
+   int m_do_extraEFdiags = 0;
+   amrex::Vector<std::unique_ptr<amrex::MultiFab>>  m_ionsFluxes;
diff --git a/Source/Efield/PeleLMEFUtils.cpp b/Source/Efield/PeleLMEFUtils.cpp
@@ -329,3 +329,21 @@ void PeleLM::fillPatchNLphiV(Real a_time,
                          refRatio(lev-1), mapper, fetchBCRecArray(PHIV,1), 0);
    }
 }
+
+void PeleLM::ionsBalance()
+{
+   // Compute the sum of ions on the domain boundaries
+   Array<Real,2*AMREX_SPACEDIM> ionsCurrent{0.0};
+   for (int n = NUM_SPECIES-NUM_IONS; n < NUM_SPECIES; n++){
+      for (int i = 0; i < 2*AMREX_SPACEDIM; i++) {
+         ionsCurrent[i] += m_domainRhoYFlux[2*n*AMREX_SPACEDIM+i] * zk[n];
+      }
+   }
+
+   tmpIonsFile << m_nstep << " " << m_cur_time;                         // Time info
+   for (int i = 0; i < 2*AMREX_SPACEDIM; i++) {
+       tmpIonsFile << " " << ionsCurrent[i];                            // ions current as xlo, xhi, ylo, ...
+   }
+   tmpIonsFile << "\n";
+   tmpIonsFile.flush();
+}
diff --git a/Source/PeleLM.H b/Source/PeleLM.H
@@ -905,12 +905,12 @@ class PeleLM : public amrex::AmrCore {
    void jTimesV(const amrex::Vector<amrex::MultiFab*> &a_x,
                 const amrex::Vector<amrex::MultiFab*> &a_Ax);
 
-
-
    void setUpPrecond(const amrex::Real &a_dt,
                      const amrex::Vector<const amrex::MultiFab*> &a_nE);
    void applyPrecond(const amrex::Vector<amrex::MultiFab*> &a_v,
                      const amrex::Vector<amrex::MultiFab*> &a_Pv);
+
+   void ionsBalance();
    //-----------------------------------------------------------------------------
 #endif
 

diff --git a/Source/PeleLMAdvection.cpp b/Source/PeleLMAdvection.cpp
@@ -639,6 +639,22 @@ void PeleLM::computeScalarAdvTerms(std::unique_ptr<AdvanceAdvData> &advData)
    // Compute face domain integral for U at every SDC iteration
    addUmacFluxes(advData, geom[0]);
 
+#ifdef PELE_USE_EFIELD
+   if (m_do_extraEFdiags) {
+      for (int lev = 0; lev <= finest_level; ++lev) {
+         for (int n = 0; n < NUM_IONS; ++n) {
+            int spec_idx = NUM_SPECIES - NUM_IONS + n;
+            Array<std::unique_ptr<MultiFab>,AMREX_SPACEDIM> ionFlux;
+            for (int idim = 0; idim < AMREX_SPACEDIM; idim++ ) {
+               ionFlux[idim].reset(new MultiFab(fluxes[lev][idim],amrex::make_alias,spec_idx,1));
+            }
+            average_face_to_cellcenter(*m_ionsFluxes[lev],n*AMREX_SPACEDIM,
+                                       GetArrOfConstPtrs(ionFlux));
+         }
+      }
+   }
+#endif
+
    //----------------------------------------------------------------
    // Fluxes divergence to get the scalars advection term
    auto AdvTypeAll = fetchAdvTypeArray(FIRSTSPEC,NUM_SPECIES+1); // Species+RhoH

diff --git a/Source/PeleLMInit.cpp b/Source/PeleLMInit.cpp
@@ -70,6 +70,9 @@ void PeleLM::MakeNewLevelFromScratch( int lev,
 
 #ifdef PELE_USE_EFIELD
    m_leveldatanlsolve[lev].reset(new LevelDataNLSolve(grids[lev], dmap[lev], *m_factory[lev], m_nGrowState));
+   if (m_do_extraEFdiags) {
+      m_ionsFluxes[lev].reset(new MultiFab(grids[lev], dmap[lev], NUM_IONS*AMREX_SPACEDIM, 0));
+   }
 #endif
 
    // Fill the initial solution (if not restarting)

diff --git a/Source/PeleLMPlot.cpp b/Source/PeleLMPlot.cpp
@@ -99,6 +99,12 @@ void PeleLM::WritePlotFile() {
    }
 #endif
 
+#ifdef PELE_USE_EFIELD
+   if (m_do_extraEFdiags) {
+       ncomp += NUM_IONS * AMREX_SPACEDIM;
+   }
+#endif
+
    //----------------------------------------------------------------
    // Plot MultiFabs
    Vector<MultiFab> mf_plt(finest_level + 1);
@@ -187,6 +193,17 @@ void PeleLM::WritePlotFile() {
    }
 #endif
 
+#ifdef PELE_USE_EFIELD
+   if (m_do_extraEFdiags) {
+       for (int ivar = 0; ivar < NUM_IONS; ++ivar) {
+           for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
+               std::string dir = (idim == 0) ? "X" : ( (idim == 1) ? "Y" : "Z");
+               plt_VarsName.push_back("DriftFlux_"+names[NUM_SPECIES-NUM_IONS+ivar]+"_"+dir);
+           }
+       }
+   }
+#endif
+
    //----------------------------------------------------------------
    // Fill the plot MultiFabs
    for (int lev = 0; lev <= finest_level; ++lev) {
@@ -266,6 +283,11 @@ void PeleLM::WritePlotFile() {
         cnt += num_spray_derive;
       }
 #endif
+#ifdef PELE_USE_EFIELD
+      if (m_do_extraEFdiags) {
+          MultiFab::Copy(mf_plt[lev], *m_ionsFluxes[lev], 0, cnt, m_ionsFluxes[lev]->nComp(),0);
+      }
+#endif
 #ifdef AMREX_USE_EB
       EB_set_covered(mf_plt[lev],0.0);
 #endif

diff --git a/Source/PeleLMRegrid.cpp b/Source/PeleLMRegrid.cpp
@@ -86,6 +86,9 @@ void PeleLM::MakeNewLevelFromCoarse( int lev,
 
 #ifdef PELE_USE_EFIELD
    m_leveldatanlsolve[lev].reset(new LevelDataNLSolve(ba, dm, *m_factory[lev], m_nGrowState));
+   if (m_do_extraEFdiags) { 
+      m_ionsFluxes[lev].reset(new MultiFab(ba, dm, NUM_IONS*AMREX_SPACEDIM, 0));
+   }
    m_precond_op.reset();
 #endif
 
@@ -173,6 +176,9 @@ void PeleLM::RemakeLevel( int lev,
 
 #ifdef PELE_USE_EFIELD
    m_leveldatanlsolve[lev].reset(new LevelDataNLSolve(ba, dm, *m_factory[lev], m_nGrowState));
+   if (m_do_extraEFdiags) { 
+      m_ionsFluxes[lev].reset(new MultiFab(ba, dm, NUM_IONS*AMREX_SPACEDIM, 0));
+   }
    m_precond_op.reset();
 #endif
 
@@ -204,6 +210,9 @@ void PeleLM::ClearLevel(int lev) {
    macproj.reset();
 #ifdef PELE_USE_EFIELD
    m_leveldatanlsolve[lev].reset();
+   if (m_do_extraEFdiags) { 
+      m_ionsFluxes[lev].reset();
+   }
 #endif
    m_extSource[lev]->clear();
 }

diff --git a/Source/PeleLMSetup.cpp b/Source/PeleLMSetup.cpp
@@ -464,7 +464,18 @@ void PeleLM::readParameters() {
    ppef.query("restart_nonEF",m_restart_nonEF);
    ppef.query("restart_electroneutral",m_restart_electroneutral);
    ppef.query("restart_resetTime",m_restart_resetTime);
+   ppef.query("plot_extras",m_do_extraEFdiags);
+
+   // Getting the ions fluxes on the domain boundaries
+   // Species balance data is needed, so override if not activated
+   if (m_do_temporals) {
+      ppef.query("do_ionsBalance",m_do_ionsBalance);
+      if (m_do_ionsBalance) {
+         m_do_speciesBalance = 1; 
+      }
+   }
 #endif
+
 #ifdef PELELM_USE_SPRAY
    readSprayParameters();
 #endif
@@ -947,6 +958,7 @@ void PeleLM::resizeArray() {
 
 #ifdef PELE_USE_EFIELD
    m_leveldatanlsolve.resize(max_level+1);
+   m_ionsFluxes.resize(max_level+1);
 #endif
 
    // External sources

diff --git a/Source/PeleLMTemporals.cpp b/Source/PeleLMTemporals.cpp
@@ -425,6 +425,13 @@ void PeleLM::writeTemporals()
    tmpExtremasFile << " \n";
    tmpExtremasFile.flush();
 
+
+#ifdef PELE_USE_EFIELD
+   if (m_do_ionsBalance) {
+      ionsBalance();
+   }
+#endif
+
 }
 
 void PeleLM::openTempFile()
@@ -453,6 +460,13 @@ void PeleLM::openTempFile()
          tmpExtremasFile.open(tempFileName.c_str(),std::ios::out | std::ios::app | std::ios_base::binary);
          tmpExtremasFile.precision(12);
       }
+#ifdef PELE_USE_EFIELD
+      if (m_do_ionsBalance) {
+         tempFileName = "temporals/tempIons";
+         tmpIonsFile.open(tempFileName.c_str(),std::ios::out | std::ios::app | std::ios_base::binary);
+         tmpIonsFile.precision(12);
+      }
+#endif
    }
 }
 
@@ -475,5 +489,11 @@ void PeleLM::closeTempFile()
          tmpExtremasFile.flush();
          tmpExtremasFile.close();
       }
+#ifdef PELE_USE_EFIELD
+      if (m_do_ionsBalance) {
+         tmpIonsFile.flush();
+         tmpIonsFile.close();
+      }
+#endif
    }
 }