Implement closed chamber algorithm. (#13)

* Add auto-detection of closed chamber and unable overwrite. Add PPquery of linear solves tolerances. * Add GammmaInv kernel. * Move MFSum in Utils and initialize uncovered volume computation. * MAC projection function handles closed chamber corrections. * Nodal projection functions handle -/+ of Sbar in RHS. * Remove TODO comment. * Pass dp0dt in diffusion forcing. * Pass dp0dt in Advection forcing. * Add pOld <-> pNew in advance function. * Add adjustPandDivU in Eos file. * Add accessor to divU levels vector * Add declarations. * Add ambient pressure to checkpoint file header. * Uses pNew to get dPdt. * Fix BL_PROF in UMAC * Add an enclosed flame test to test closed chamber. * Add a CI testing closed chamber.
AMReX-Combustion · Sep 27, 2021 · e4a70d4 · e4a70d4
1 parent 7cd8cc5
commit e4a70d4
Show file tree

Hide file tree

Showing 20 changed files with 2,703 additions and 110 deletions.
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -119,3 +119,34 @@ jobs:
       working-directory: ./Exec/RegTests/PeriodicCases/
       run: |
         ./PeleLMeX2d.gnu.ex input.2d_CoVo amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
+
+  # Build and Run the EnclosedFlame RegTest with GNU7.5 and MPI support
+  EF2D_MPIRun:
+    name: GNU@7.5 MPI Run [EF2D]
+    runs-on: ubuntu-latest
+    env: 
+      {CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
+    steps:
+    - name: Cancel previous runs
+      uses: styfle/cancel-workflow-action@0.6.0
+      with:
+        access_token: ${{github.token}}
+    - uses: actions/checkout@v2
+    - name: System Dependencies
+      run: .github/workflows/dependencies/dependencies.sh
+    - name: Repo Dependencies
+      run: Utils/CloneDeps.sh
+    - name: Build
+      env:
+         AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex
+         PELE_PHYSICS_HOME: ${GITHUB_WORKSPACE}/build/PelePhysics
+         PELELM_HOME: ${GITHUB_WORKSPACE}
+         AMREX_HYDRO_HOME: ${GITHUB_WORKSPACE}/build/AMReX-Hydro
+      working-directory: ./Exec/RegTests/EnclosedFlame/
+      run: |
+        make TPL COMP=gnu USE_MPI=TRUE
+        make -j 2 COMP=gnu USE_MPI=TRUE
+    - name: Run
+      working-directory: ./Exec/RegTests/EnclosedFlame/
+      run: |
+        ./PeleLMeX2d.gnu.MPI.ex input.2d-regt amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
diff --git a/Exec/RegTests/EnclosedFlame/GNUmakefile b/Exec/RegTests/EnclosedFlame/GNUmakefile
@@ -0,0 +1,29 @@
+TOP = ../../../..
+AMREX_HOME         ?= ${TOP}/amrex
+PELELMEX_HOME      ?= ${TOP}/PeleLMeX
+PELE_PHYSICS_HOME  ?= ${TOP}/PelePhysics
+AMREX_HYDRO_HOME   ?= ${TOP}/AMReX-Hydro
+
+
+# AMReX
+DIM             = 2
+DEBUG           = FALSE
+PRECISION       = DOUBLE
+VERBOSE         = FALSE
+TINY_PROFILE    = FALSE
+
+# Compilation
+COMP            = gnu
+USE_MPI         = TRUE
+USE_OMP         = FALSE
+USE_CUDA        = FALSE
+USE_HIP         = FALSE
+
+# PeleLMeX
+
+# PelePhysics
+Chemistry_Model = drm19
+Eos_Model		= Fuego
+Transport_Model = Simple
+
+include $(PELELMEX_HOME)/Utils/Make.PeleLMeX
diff --git a/Exec/RegTests/EnclosedFlame/drm19_pmf.dat b/Exec/RegTests/EnclosedFlame/drm19_pmf.dat
diff --git a/Exec/RegTests/EnclosedFlame/input.2d-regt b/Exec/RegTests/EnclosedFlame/input.2d-regt
@@ -0,0 +1,86 @@
+#----------------------DOMAIN DEFINITION------------------------
+geometry.is_periodic = 0 0               # For each dir, 0: non-perio, 1: periodic
+geometry.coord_sys   = 0                  # 0 => cart, 1 => RZ
+geometry.prob_lo     = -0.02 -0.02 0.0        # x_lo y_lo (z_lo)
+geometry.prob_hi     =  0.02  0.02 0.016        # x_hi y_hi (z_hi)
+
+# >>>>>>>>>>>>>  BC FLAGS <<<<<<<<<<<<<<<<
+# Interior, Inflow, Outflow, Symmetry,
+# SlipWallAdiab, NoSlipWallAdiab, SlipWallIsotherm, NoSlipWallIsotherm
+peleLM.lo_bc = NoSlipWallAdiab NoSlipWallAdiab
+peleLM.hi_bc = NoSlipWallAdiab NoSlipWallAdiab
+
+
+#-------------------------AMR CONTROL----------------------------
+amr.n_cell          = 128 128 32      # Level 0 number of cells in each direction   
+amr.v               = 1                # AMR verbose
+amr.max_level       = 2                # maximum level number allowed
+amr.ref_ratio       = 2 2 2 2 2         # refinement ratio
+amr.regrid_int      = 5                # how often to regrid
+amr.n_error_buf     = 2 2 2 2          # number of buffer cells in error est
+amr.grid_eff        = 0.7              # what constitutes an efficient grid
+amr.blocking_factor = 16               # block factor in grid generation (min box size)
+amr.max_grid_size   = 128               # max box size
+
+
+#--------------------------- Problem -------------------------------
+prob.P_mean = 101325.0
+prob.standoff = -0.002
+prob.pertmag = 0.0002
+prob.pmf_datafile = "drm19_pmf.dat"
+
+#-------------------------PeleLM CONTROL----------------------------
+peleLM.v = 1
+peleLM.incompressible = 0
+peleLM.rho = 1.17
+peleLM.mu = 0.0
+peleLM.use_wbar = 1
+peleLM.sdc_iterMax = 2
+peleLM.floor_species = 0
+peleLM.num_divu_iter = 1
+peleLM.num_init_iter = 3
+
+peleLM.do_temporals = 1
+peleLM.temporal_int = 2
+peleLM.mass_balance = 1
+
+#amr.restart = chk00005
+amr.check_int = 5
+amr.plot_int = 100
+amr.max_step = 10
+amr.dt_shrink = 0.0001
+amr.stop_time = 1.0
+#amr.stop_time = 1.00
+amr.cfl = 0.15
+amr.derive_plot_vars = avg_pressure mag_vort mass_fractions
+
+# --------------- INPUTS TO CHEMISTRY REACTOR ---------------
+peleLM.chem_integrator = "ReactorCvode"
+#peleLM.use_typ_vals_chem = 1          # Use species/temp typical values in CVODE
+#ode.rtol = 1.0e-6                     # Relative tolerance of the chemical solve
+#ode.atol = 1.0e-5                     # Absolute tolerance factor applied on typical values
+cvode.solve_type = denseAJ_direct     # CVODE Linear solve type (for Newton direction) 
+cvode.max_order  = 4                  # CVODE max BDF order. 
+
+#--------------------REFINEMENT CONTROL------------------------
+#amr.refinement_indicators = temp
+#amr.temp.max_level     = 1
+#amr.temp.value_greater = 305
+#amr.temp.field_name    = temp
+
+#amr.refinement_indicators = magVort
+#amr.magVort.max_level     = 1
+#amr.magVort.value_greater = 500.0
+#amr.magVort.field_name    = mag_vort
+
+amr.refinement_indicators = yH
+amr.yH.max_level     = 5
+amr.yH.value_greater = 3.0e-7
+amr.yH.field_name    = Y(H)
+
+#--------------------LINEAR SOLVER CONTROL------------------------
+nodal_proj.verbose = 0
+nodal_proj.rtol = 1.0e-11
+amrex.fpe_trap_invalid = 1
+amrex.fpe_trap_zero = 1
+amrex.fpe_trap_overflow = 1
diff --git a/Exec/RegTests/EnclosedFlame/pelelm_prob.H b/Exec/RegTests/EnclosedFlame/pelelm_prob.H
@@ -0,0 +1,142 @@
+#ifndef PELELM_PROB_H_
+#define PELELM_PROB_H_
+
+#include <AMReX_Geometry.H>
+#include <AMReX_FArrayBox.H>
+#include <AMReX_SPACE.H>
+
+#include <PeleLM_Index.H>
+#include <pelelm_prob_parm.H>
+#include <pmf.H>
+#include <pmf_data.H>
+#include <PelePhysics.H>
+
+AMREX_GPU_DEVICE
+AMREX_FORCE_INLINE
+void pelelm_initdata(int i, int j, int k,
+                     int is_incompressible,
+                     amrex::Array4<amrex::Real> const& vel,
+                     amrex::Array4<amrex::Real> const& rho,
+                     amrex::Array4<amrex::Real> const& rhoY,
+                     amrex::Array4<amrex::Real> const& rhoH,
+                     amrex::Array4<amrex::Real> const& temp,
+                     amrex::Array4<amrex::Real> const& aux,
+                     amrex::GeometryData const& geomdata,
+                     ProbParm const& prob_parm,
+                     PmfData const * pmf_data)
+{
+    const amrex::Real* prob_lo = geomdata.ProbLo();
+    const amrex::Real* prob_hi = geomdata.ProbHi();
+    const amrex::Real* dx      = geomdata.CellSize();
+
+    AMREX_D_TERM(const amrex::Real x = prob_lo[0] + (i+0.5)*dx[0];,
+                 const amrex::Real y = prob_lo[1] + (j+0.5)*dx[1];,
+                 const amrex::Real z = prob_lo[2] + (k+0.5)*dx[2];);
+
+    AMREX_D_TERM(const amrex::Real Lx = prob_hi[0] - prob_lo[0];,
+                 const amrex::Real Ly = prob_hi[1] - prob_lo[1];,
+                 const amrex::Real Lz = prob_hi[2] - prob_lo[2]);
+
+    AMREX_D_TERM(const amrex::Real xc = prob_lo[0] + 0.5 * Lx;,
+                 const amrex::Real yc = prob_lo[1] + 0.5 * Ly;,
+                 const amrex::Real zc = prob_lo[2] + 0.5 * Lz;);
+
+    constexpr amrex::Real Pi = 3.14159265358979323846264338327950288;
+
+    auto eos = pele::physics::PhysicsType::eos();
+    amrex::GpuArray<amrex::Real, NUM_SPECIES + 4> pmf_vals = {0.0};
+    amrex::Real molefrac[NUM_SPECIES] = {0.0};
+    amrex::Real massfrac[NUM_SPECIES] = {0.0};
+    amrex::Real pert = 0.0;
+
+    amrex::Real radius = std::sqrt(AMREX_D_TERM(  (x-xc)*(x-xc),
+                                                + (y-yc)*(y-yc),
+                                                + (z-zc)*(z-zc)));
+
+    if (prob_parm.pertmag > 0.0) 
+    {
+
+#if ( AMREX_SPACEDIM == 2 )
+       amrex::Real angle = std::atan2((y-yc),(x-xc));
+       pert = prob_parm.pertmag * 
+             (0.7 * std::sin(2*angle) +
+              //1.023 * std::sin(2 * (angle - 0.004598) / radius2) +
+              //0.945 * std::sin(3 * (angle - 0.00712435) / radius2) +
+              0.9 * std::sin(8 * angle ) +
+              1.4 * std::sin(4 * angle  ));
+    }
+
+    amrex::Real y1 = 3.6 - (radius - prob_parm.standoff - 1.42*dx[1] + pert)*100;
+    amrex::Real y2 = 3.6 - (radius - prob_parm.standoff + 1.42*dx[1] + pert)*100;
+
+#elif ( AMREX_SPACEDIM == 3 )
+
+       pert = prob_parm.pertmag *
+             (1.0 * std::sin(2 * Pi * 4 * x / Lx) *
+                std::sin(2 * Pi * 5 * y / Ly) +
+              1.023 * std::sin(2 * Pi * 2 * (x - 0.004598) / Lx) *
+                std::sin(2 * Pi * 4 * (y - 0.0053765) / Ly) +
+              0.945 * std::sin(2 * Pi * 3 * (x - 0.00712435) / Lx) *
+                std::sin(2 * Pi * 3 * (y - 0.02137) / Ly) +
+              1.017 * std::sin(2 * Pi * 5 * (x - 0.0033) / Lx) *
+                std::sin(2 * Pi * 6 * (y - 0.018) / Ly) +
+              0.982 * std::sin(2 * Pi * 5 * (x - 0.014234) / Lx));
+    }
+
+    amrex::Real y1 = (z - prob_parm.standoff - 0.5*dx[2] + pert)*100;
+    amrex::Real y2 = (z - prob_parm.standoff + 0.5*dx[2] + pert)*100;
+#endif
+
+
+    PMF::pmf(pmf_data,y1, y1, pmf_vals);
+
+    temp(i,j,k) = pmf_vals[0];;
+
+    for (int n = 0; n < NUM_SPECIES; n++){
+      molefrac[n] = pmf_vals[3 + n];
+    }
+    eos.X2Y(molefrac, massfrac);
+
+    vel(i,j,k,0) = 0.0;
+#if ( AMREX_SPACEDIM == 2 )
+    vel(i,j,k,1) = pmf_vals[1]*1e-2;
+#elif ( AMREX_SPACEDIM == 3 )
+    vel(i,j,k,1) = 0.0;
+    vel(i,j,k,2) = pmf_vals[1]*1e-2;
+#endif
+
+    amrex::Real P_cgs = prob_parm.P_mean * 10.0;
+
+    // Density
+    amrex::Real rho_cgs = 0.0;
+    eos.PYT2R(P_cgs, massfrac, temp(i,j,k), rho_cgs);
+    rho(i,j,k) = rho_cgs * 1.0e3;
+
+    // Enthalpy
+    amrex::Real h_cgs = 0.0;
+    eos.TY2H(temp(i,j,k), massfrac, h_cgs);
+    rhoH(i,j,k) = h_cgs * 1.0e-4 * rho(i,j,k);
+
+    // Species mass
+    for (int n = 0; n < NUM_SPECIES; n++) {
+       rhoY(i,j,k,n) = massfrac[n] * rho(i,j,k);
+    }
+}
+
+AMREX_GPU_DEVICE
+AMREX_FORCE_INLINE
+void
+bcnormal(
+  const amrex::Real* /*x[AMREX_SPACEDIM]*/,
+  const int m_nAux,
+  amrex::Real s_ext[NVAR],
+  const int idir,
+  const int sgn,
+  const amrex::Real time,
+  amrex::GeometryData const& geomdata,
+  ProbParm const& prob_parm,
+  PmfData const *pmf_data)
+{
+  const amrex::Real* prob_lo = geomdata.ProbLo();
+}
+#endif
diff --git a/Exec/RegTests/EnclosedFlame/pelelm_prob.cpp b/Exec/RegTests/EnclosedFlame/pelelm_prob.cpp
@@ -0,0 +1,18 @@
+#include <PeleLM.H>
+#include <AMReX_ParmParse.H>
+#include <pmf.H>
+
+void PeleLM::readProbParm()
+{
+   amrex::ParmParse pp("prob");
+
+   std::string type;
+   pp.query("P_mean",   PeleLM::prob_parm->P_mean);
+   pp.query("standoff", PeleLM::prob_parm->standoff);
+   pp.query("pertmag",  PeleLM::prob_parm->pertmag);
+
+   std::string pmf_datafile;
+   pp.query("pmf_datafile", pmf_datafile);
+   int pmf_do_average = 0;
+   PMF::read_pmf(pmf_datafile, pmf_do_average);
+}
diff --git a/Exec/RegTests/EnclosedFlame/pelelm_prob_parm.H b/Exec/RegTests/EnclosedFlame/pelelm_prob_parm.H
@@ -0,0 +1,17 @@
+#ifndef PELELM_PROB_PARM_H_
+#define PELELM_PROB_PARM_H_
+
+#include <AMReX_REAL.H>
+#include <AMReX_GpuMemory.H>
+
+using namespace amrex::literals;
+
+struct ProbParm
+    : amrex::Gpu::Managed
+{
+   amrex::Real P_mean   = 101325.0_rt;
+   amrex::Real standoff = 0.0_rt;
+   amrex::Real pertmag  = 0.0004_rt;
+   int  meanFlowDir = 1;
+};
+#endif
diff --git a/Source/PeleLM.H b/Source/PeleLM.H
@@ -401,6 +401,10 @@ class PeleLM : public amrex::AmrCore {
                   const amrex::Vector<amrex::MultiFab*> &a_dPdt);
    void calc_dPdt(int lev, const PeleLM::TimeStamp &a_time,
                   amrex::MultiFab* a_dPdt);
+
+   // Closed Chamber
+   amrex::Real
+   adjustPandDivU(std::unique_ptr<AdvanceAdvData> &advData);
    //-----------------------------------------------------------------------------
 
    //-----------------------------------------------------------------------------
@@ -724,6 +728,7 @@ class PeleLM : public amrex::AmrCore {
    amrex::Vector<amrex::MultiFab* > getDensityVect(const PeleLM::TimeStamp &a_time);
    amrex::Vector<amrex::MultiFab* > getSpeciesVect(const PeleLM::TimeStamp &a_time);
    amrex::Vector<amrex::MultiFab* > getTempVect(const PeleLM::TimeStamp &a_time);
+   amrex::Vector<amrex::MultiFab* > getDivUVect(const PeleLM::TimeStamp &a_time);
    amrex::Vector<amrex::MultiFab* > getDiffusivityVect(const PeleLM::TimeStamp &a_time);
    amrex::Vector<amrex::MultiFab* > getViscosityVect(const PeleLM::TimeStamp &a_time);
 #ifdef PLM_USE_EFIELD
@@ -883,6 +888,8 @@ class PeleLM : public amrex::AmrCore {
    amrex::Real m_dpdtFactor = 1.0;
    amrex::Real m_pOld = -1.0;
    amrex::Real m_pNew = -1.0;
+   amrex::Real m_dp0dt = 0.0;
+   amrex::Real m_uncoveredVol = -1.0;
    int m_closed_chamber = 0;
 
    // Godunov (no MOL in LMeX)