Enable EB in LMeX (#16)

* Add missing EB sources to Make.

* Add volFrac to plotfile.

* Add PeleLM EB source file.

* Add a static_cast accessor to ebfactory.

* Update AMReX-Hydro calls to pass ebfactory in.

* Missing include for volfrac tagging.

* Update diffusion operator for EB.

* EB compiles and initialize properly.

* Make sure divTau has right Factory.

* Fix getDiffusivity to not set covered face to 0,0 but huge val.

* _OPENMP -> AMREX_USE_OMP

* Remove plot/abort after init.

* First pass at updating DiffusionOp for EB. Mostly done.

* Fix divergence of advective flux function for EB.

* Minor tweak in PeleLMDiffusion. FluxRedist still need to be implemented.

* Re-organive advection to get divergnce and then redistribute when EB.
fillBoundary in between req. a temporary.

* redistribute_AofS function.

* Add redist schemes for adv/diff, ensure enough ghost cells and declarations.

* Add ParmParse to control nodal solve MG depth.

* Enforce EB-covered to 0 for nodal projection.

* Add fluxdivegenceRD to handle diffusive flux redistribution. Fix MFSum
to exclude EB-covered regions.

* Diffusive flux redistribution function.

* Tedious switch between div/divRD for diffusiob fluxes. Might want to simplify later.

* Accessor to RhoH tower.

* Declarations.

* Add 3D case for EnclosedFlame and a thermo-diffusive CH4/H2 PMF.

* Number of ghost cells in EBFactory should be 6 because of StateRedist

* EBFlag test inst. reaction rate.

* Set covered to 0.0 in plotfiles.

* EBFlag in getHGivenT in enthalpy flux.

* Do initial redisitribution and get a covered state.

* Add initial redistribution machinery and first version of covered state. Need to improve !

* Less confusing message since we now have EB.

* StateRedist -> NewStateRedist default and EB-covered declaration.

* Add EB_FlowPastCylinder case, 2D/3D.

* Set EB-covered chi_incr to zero.

* Add FlowPastCylinder CI.

.github/workflows/linux.yml

@@ -150,3 +150,35 @@ jobs:
       working-directory: ./Exec/RegTests/EnclosedFlame/
       run: |
         ./PeleLMeX2d.gnu.MPI.ex input.2d-regt amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
+
+  # Build and Run the EB_FlowPastCylinder RegTest with GNU7.5 and MPI support
+  FPC2D_MPIRun:
+    name: GNU@7.5 MPI Run [FPC2D]
+    runs-on: ubuntu-latest
+    env: 
+      {CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
+    steps:
+    - name: Cancel previous runs
+      uses: styfle/cancel-workflow-action@0.6.0
+      with:
+        access_token: ${{github.token}}
+    - uses: actions/checkout@v2
+    - name: System Dependencies
+      run: .github/workflows/dependencies/dependencies.sh
+    - name: Repo Dependencies
+      run: Utils/CloneDeps.sh
+    - name: Build
+      env:
+         AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex
+         PELE_PHYSICS_HOME: ${GITHUB_WORKSPACE}/build/PelePhysics
+         PELELM_HOME: ${GITHUB_WORKSPACE}
+         AMREX_HYDRO_HOME: ${GITHUB_WORKSPACE}/build/AMReX-Hydro
+      working-directory: ./Exec/RegTests/EB_FlowPastCylinder/
+      run: |
+        make TPL COMP=gnu USE_MPI=TRUE
+        make -j 2 COMP=gnu USE_MPI=TRUE
+    - name: Run
+      working-directory: ./Exec/RegTests/EB_FlowPastCylinder/
+      run: |
+        ./PeleLMeX2d.gnu.MPI.ex input.2d-regt amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
+        ./PeleLMeX2d.gnu.MPI.ex input.2d-regt_WallBump amr.max_step=2 amr.plot_int=-1 amr.check_int=-1

Exec/RegTests/EB_FlowPastCylinder/GNUmakefile

@@ -0,0 +1,30 @@
+TOP = ../../../..
+AMREX_HOME         ?= ${TOP}/amrex
+PELELMEX_HOME      ?= ${TOP}/PeleLMeX
+PELE_PHYSICS_HOME  ?= ${TOP}/PelePhysics
+AMREX_HYDRO_HOME   ?= ${TOP}/AMReX-Hydro
+
+
+# AMReX
+DIM             = 2
+DEBUG           = FALSE
+PRECISION       = DOUBLE
+VERBOSE         = FALSE
+TINY_PROFILE    = FALSE
+USE_EB			= TRUE
+
+# Compilation
+COMP            = gnu
+USE_MPI         = TRUE
+USE_OMP         = FALSE
+USE_CUDA        = FALSE
+USE_HIP         = FALSE
+
+# PeleLMeX
+
+# PelePhysics
+Chemistry_Model = air
+Eos_Model       = Fuego
+Transport_Model = Constant
+
+include $(PELELMEX_HOME)/Utils/Make.PeleLMeX

Exec/RegTests/EB_FlowPastCylinder/input.2d-regt

@@ -0,0 +1,66 @@
+#----------------------DOMAIN DEFINITION------------------------
+geometry.is_periodic = 0 1                # For each dir, 0: non-perio, 1: periodic
+geometry.coord_sys   = 0                  # 0 => cart, 1 => RZ
+geometry.prob_lo     = -0.02 0.0 0.0        # x_lo y_lo (z_lo)
+geometry.prob_hi     =  0.06 0.04 1.0  # x_hi y_hi (z_hi)
+
+# >>>>>>>>>>>>>  BC FLAGS <<<<<<<<<<<<<<<<
+# Interior, Inflow, Outflow, Symmetry,
+# SlipWallAdiab, NoSlipWallAdiab, SlipWallIsotherm, NoSlipWallIsotherm
+peleLM.lo_bc = Inflow  Interior NoSlipWallAdiab 
+peleLM.hi_bc = Outflow Interior NoSlipWallAdiab 
+
+
+#-------------------------AMR CONTROL----------------------------
+amr.n_cell          = 64 32 64         # Level 0 number of cells in each direction   
+amr.v               = 1                # AMR verbose
+amr.max_level       = 2                # maximum level number allowed
+amr.ref_ratio       = 2 2 2 2          # refinement ratio
+amr.regrid_int      = 2                # how often to regrid
+amr.n_error_buf     = 1 1 2 2          # number of buffer cells in error est
+amr.grid_eff        = 0.7              # what constitutes an efficient grid
+amr.blocking_factor = 16               # block factor in grid generation (min box size)
+amr.max_grid_size   = 64               # max box size
+
+#--------------------------- Problem -------------------------------
+prob.T_mean = 298.0
+prob.P_mean = 101325.0
+prob.type   = ConvectedVortex
+prob.meanFlowMag = 5.0
+prob.meanFlowDir = 1
+
+#------------  INPUTS TO CONSTANT TRANSPORT -----------------
+transport.const_viscosity        = 0.0001 
+transport.const_bulk_viscosity   = 0.0 
+transport.const_conductivity     = 0.0 
+transport.const_diffusivity      = 0.0 
+
+#-------------------------PeleLM CONTROL----------------------------
+peleLM.v = 1
+peleLM.incompressible = 0
+peleLM.rho = 1.17
+peleLM.mu = 0.0003
+
+amr.plot_int = 20
+amr.max_step = 20
+amr.dt_shrink = 0.1
+amr.stop_time = 0.02
+amr.cfl = 0.7
+amr.derive_plot_vars = avg_pressure mag_vort
+
+#------------------------- EB SETUP -----------------------------
+eb2.geom_type = sphere
+eb2.sphere_radius = 0.006
+eb2.sphere_center = 0.0 0.02
+eb2.sphere_has_fluid_inside = 0
+eb2.small_volfrac = 1.0e-4
+
+#--------------------REFINEMENT CONTROL------------------------
+#amr.refinement_indicators = temp
+#amr.temp.max_level     = 2
+#amr.temp.value_greater = 305
+#amr.temp.field_name    = temp
+
+amrex.fpe_trap_invalid = 1
+amrex.fpe_trap_zero = 1
+amrex.fpe_trap_overflow = 1

Exec/RegTests/EB_FlowPastCylinder/input.2d-regt_WallBump

@@ -0,0 +1,68 @@
+#----------------------DOMAIN DEFINITION------------------------
+geometry.is_periodic = 0 0                # For each dir, 0: non-perio, 1: periodic
+geometry.coord_sys   = 0                  # 0 => cart, 1 => RZ
+geometry.prob_lo     = -0.02 0.0 0.0        # x_lo y_lo (z_lo)
+geometry.prob_hi     =  0.06 0.04 1.0  # x_hi y_hi (z_hi)
+
+# >>>>>>>>>>>>>  BC FLAGS <<<<<<<<<<<<<<<<
+# Interior, Inflow, Outflow, Symmetry,
+# SlipWallAdiab, NoSlipWallAdiab, SlipWallIsotherm, NoSlipWallIsotherm
+peleLM.lo_bc = Inflow  NoSlipWallAdiab
+peleLM.hi_bc = Outflow Symmetry
+
+#-------------------------AMR CONTROL----------------------------
+amr.n_cell          = 64 32 64         # Level 0 number of cells in each direction   
+amr.v               = 1                # AMR verbose
+amr.max_level       = 2                # maximum level number allowed
+amr.ref_ratio       = 2 2 2 2          # refinement ratio
+amr.regrid_int      = 2                # how often to regrid
+amr.n_error_buf     = 2 2 2 2          # number of buffer cells in error est
+amr.grid_eff        = 0.7              # what constitutes an efficient grid
+amr.blocking_factor = 16               # block factor in grid generation (min box size)
+amr.max_grid_size   = 64               # max box size
+
+#--------------------------- Problem -------------------------------
+prob.T_mean = 298.0
+prob.P_mean = 101325.0
+prob.type   = ConvectedVortex
+prob.meanFlowMag = 5.0
+prob.meanFlowDir = 1
+
+#------------  INPUTS TO CONSTANT TRANSPORT -----------------
+transport.const_viscosity        = 0.0001 
+transport.const_bulk_viscosity   = 0.0 
+transport.const_conductivity     = 0.0 
+transport.const_diffusivity      = 0.0 
+
+#-------------------------PeleLM CONTROL----------------------------
+peleLM.v = 3
+peleLM.incompressible = 0
+peleLM.rho = 1.17
+peleLM.mu = 0.0003
+
+amr.plot_int = 20
+amr.max_step = 20
+amr.dt_shrink = 0.1
+amr.stop_time = 0.02
+amr.cfl = 0.7
+amr.derive_plot_vars = avg_pressure mag_vort
+
+#------------------------- EB SETUP -----------------------------
+eb2.geom_type = sphere
+eb2.sphere_radius = 0.006
+eb2.sphere_center = 0.0 0.00
+eb2.sphere_has_fluid_inside = 0
+#eb2.small_volfrac = 1.0e-4
+
+#--------------------REFINEMENT CONTROL------------------------
+#amr.refinement_indicators = vort_low vort_high
+#amr.vort_low.max_level     = 2
+#amr.vort_low.value_less    = -2000
+#amr.vort_low.field_name    = mag_vort
+#amr.vort_high.max_level     = 3
+#amr.vort_high.value_greater = 2000
+#amr.vort_high.field_name    = mag_vort
+
+amrex.fpe_trap_invalid = 1
+amrex.fpe_trap_zero = 1
+amrex.fpe_trap_overflow = 1

Exec/RegTests/EB_FlowPastCylinder/input.3d-regt

@@ -0,0 +1,70 @@
+#----------------------DOMAIN DEFINITION------------------------
+geometry.is_periodic = 0 1 1                # For each dir, 0: non-perio, 1: periodic
+geometry.coord_sys   = 0                  # 0 => cart, 1 => RZ
+geometry.prob_lo     = -0.02 0.0  0.0        # x_lo y_lo (z_lo)
+geometry.prob_hi     =  0.06 0.04 0.02  # x_hi y_hi (z_hi)
+
+# >>>>>>>>>>>>>  BC FLAGS <<<<<<<<<<<<<<<<
+# Interior, Inflow, Outflow, Symmetry,
+# SlipWallAdiab, NoSlipWallAdiab, SlipWallIsotherm, NoSlipWallIsotherm
+peleLM.lo_bc = Inflow  Interior Interior
+peleLM.hi_bc = Outflow Interior Interior
+
+
+#-------------------------AMR CONTROL----------------------------
+amr.n_cell          = 64 32 16         # Level 0 number of cells in each direction   
+amr.v               = 1                # AMR verbose
+amr.max_level       = 2                # maximum level number allowed
+amr.ref_ratio       = 2 2 2 2          # refinement ratio
+amr.regrid_int      = 2                # how often to regrid
+amr.n_error_buf     = 1 1 2 2          # number of buffer cells in error est
+amr.grid_eff        = 0.7              # what constitutes an efficient grid
+amr.blocking_factor = 16               # block factor in grid generation (min box size)
+amr.max_grid_size   = 64               # max box size
+
+#--------------------------- Problem -------------------------------
+prob.T_mean = 298.0
+prob.P_mean = 101325.0
+prob.type   = ConvectedVortex
+prob.meanFlowMag = 5.0
+prob.meanFlowDir = 1
+
+#------------  INPUTS TO CONSTANT TRANSPORT -----------------
+transport.const_viscosity        = 0.0001 
+transport.const_bulk_viscosity   = 0.0 
+transport.const_conductivity     = 0.0 
+transport.const_diffusivity      = 0.0 
+
+#-------------------------PeleLM CONTROL----------------------------
+peleLM.v = 1
+peleLM.incompressible = 0
+peleLM.rho = 1.17
+peleLM.mu = 0.0003
+
+amr.plot_int = 20
+amr.max_step = 5
+amr.dt_shrink = 0.1
+amr.stop_time = 0.02
+amr.cfl = 0.7
+amr.derive_plot_vars = avg_pressure mag_vort
+
+#------------------------- EB SETUP -----------------------------
+eb2.geom_type = cylinder
+eb2.cylinder_radius     = 0.006
+eb2.cylinder_direction  = 2
+eb2.cylinder_center     = 0.0 0.02 0.0
+eb2.cylinder.internal_flow    = false
+eb2.cylinder_has_fluid_inside = 0
+eb2.small_volfrac = 1.0e-4
+
+#--------------------REFINEMENT CONTROL------------------------
+#amr.refinement_indicators = temp
+#amr.temp.max_level     = 2
+#amr.temp.value_greater = 305
+#amr.temp.field_name    = temp
+#
+fabarray.mfiter_tile_size = 1024 1024 1024
+
+amrex.fpe_trap_invalid = 1
+amrex.fpe_trap_zero = 1
+amrex.fpe_trap_overflow = 1

Exec/RegTests/EB_FlowPastCylinder/input.3d-regtY

@@ -0,0 +1,70 @@
+#----------------------DOMAIN DEFINITION------------------------
+geometry.is_periodic = 1 0 1                # For each dir, 0: non-perio, 1: periodic
+geometry.coord_sys   = 0                  # 0 => cart, 1 => RZ
+geometry.prob_lo     = 0.00 -0.02  0.0        # x_lo y_lo (z_lo)
+geometry.prob_hi     = 0.04  0.06 0.02  # x_hi y_hi (z_hi)
+
+# >>>>>>>>>>>>>  BC FLAGS <<<<<<<<<<<<<<<<
+# Interior, Inflow, Outflow, Symmetry,
+# SlipWallAdiab, NoSlipWallAdiab, SlipWallIsotherm, NoSlipWallIsotherm
+peleLM.lo_bc = Interior Inflow  Interior
+peleLM.hi_bc = Interior Outflow Interior
+
+
+#-------------------------AMR CONTROL----------------------------
+amr.n_cell          = 32 64 16         # Level 0 number of cells in each direction   
+amr.v               = 1                # AMR verbose
+amr.max_level       = 2                # maximum level number allowed
+amr.ref_ratio       = 2 2 2 2          # refinement ratio
+amr.regrid_int      = 2                # how often to regrid
+amr.n_error_buf     = 1 1 2 2          # number of buffer cells in error est
+amr.grid_eff        = 0.7              # what constitutes an efficient grid
+amr.blocking_factor = 16               # block factor in grid generation (min box size)
+amr.max_grid_size   = 64               # max box size
+
+#--------------------------- Problem -------------------------------
+prob.T_mean = 298.0
+prob.P_mean = 101325.0
+prob.type   = ConvectedVortex
+prob.meanFlowMag = 5.0
+prob.meanFlowDir = 2
+
+#------------  INPUTS TO CONSTANT TRANSPORT -----------------
+transport.const_viscosity        = 0.0001 
+transport.const_bulk_viscosity   = 0.0 
+transport.const_conductivity     = 0.0 
+transport.const_diffusivity      = 0.0 
+
+#-------------------------PeleLM CONTROL----------------------------
+peleLM.v = 1
+peleLM.incompressible = 0
+peleLM.rho = 1.17
+peleLM.mu = 0.0003
+
+amr.plot_int = 20
+amr.max_step = 5
+amr.dt_shrink = 0.1
+amr.stop_time = 0.02
+amr.cfl = 0.7
+amr.derive_plot_vars = avg_pressure mag_vort
+
+#------------------------- EB SETUP -----------------------------
+eb2.geom_type = cylinder
+eb2.cylinder_radius     = 0.006
+eb2.cylinder_direction  = 2
+eb2.cylinder_center     = 0.02 0.0 0.0
+eb2.cylinder.internal_flow    = false
+eb2.cylinder_has_fluid_inside = 0
+eb2.small_volfrac = 1.0e-4
+
+#--------------------REFINEMENT CONTROL------------------------
+#amr.refinement_indicators = temp
+#amr.temp.max_level     = 2
+#amr.temp.value_greater = 305
+#amr.temp.field_name    = temp
+#
+fabarray.mfiter_tile_size = 1024 1024 1024
+
+amrex.fpe_trap_invalid = 1
+amrex.fpe_trap_zero = 1
+amrex.fpe_trap_overflow = 1