Update TurbInflow

.github/workflows/linux.yml

@@ -307,3 +307,47 @@ jobs:
       working-directory: ./Exec/RegTests/EB_PipeFlow/
       run: |
         ./PeleLMeX3d.gnu.MPI.ex input.3d-regt amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
+
+  # Build and Run the TurbInflow RegTest in DEBUG with GNU7.5 and MPI support
+  TI3D_MPIRun:
+    name: GNU@7.5 MPI Run [TI3DDBG]
+    runs-on: ubuntu-latest
+    env: 
+      {CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
+    steps:
+    - name: Cancel previous runs
+      uses: styfle/cancel-workflow-action@0.6.0
+      with:
+        access_token: ${{github.token}}
+    - uses: actions/checkout@v2
+    - name: System Dependencies
+      run: .github/workflows/dependencies/dependencies.sh
+    - name: Repo Dependencies
+      run: Utils/CloneDeps.sh
+    - name: GenerateTurbFile
+      env:
+         AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex
+         PELE_PHYSICS_HOME: ${GITHUB_WORKSPACE}/build/PelePhysics
+         PELELM_HOME: ${GITHUB_WORKSPACE}
+         AMREX_HYDRO_HOME: ${GITHUB_WORKSPACE}/build/AMReX-Hydro
+      working-directory: ./Exec/RegTests/TurbInflow/TurbFileHIT
+      run: |
+        make -j 2 COMP=gnu
+        ./PeleTurb3d.gnu.ex input hit_file=../../HITDecay/hit_ic_4_32.dat input_ncell=32
+    - name: Build
+      env:
+         AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex
+         PELE_PHYSICS_HOME: ${GITHUB_WORKSPACE}/build/PelePhysics
+         PELELM_HOME: ${GITHUB_WORKSPACE}
+         AMREX_HYDRO_HOME: ${GITHUB_WORKSPACE}/build/AMReX-Hydro
+      working-directory: ./Exec/RegTests/TurbInflow/
+      run: |
+        make TPL COMP=gnu USE_MPI=TRUE DEBUG=TRUE
+        make -j 2 COMP=gnu USE_MPI=TRUE DEBUG=TRUE
+    - name: Run
+      working-directory: ./Exec/RegTests/TurbInflow/
+      run: |
+        mpirun -n 2 ./PeleLMeX3d.gnu.DEBUG.MPI.ex input.3d peleLM.num_init_iter=1 amr.n_cell=32 32 32 amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
+        mpirun -n 2 ./PeleLMeX3d.gnu.DEBUG.MPI.ex input.3d_negY peleLM.num_init_iter=1 amr.n_cell=32 32 32 amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
+        mpirun -n 2 ./PeleLMeX3d.gnu.DEBUG.MPI.ex input.3d_posX peleLM.num_init_iter=1 amr.n_cell=32 32 32 amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
+        mpirun -n 2 ./PeleLMeX3d.gnu.DEBUG.MPI.ex input.3d_BoxLoZ peleLM.num_init_iter=1 amr.n_cell=32 32 32 amr.max_step=2 amr.plot_int=-1 amr.check_int=-1

Exec/RegTests/TurbInflow/GNUmakefile

@@ -20,7 +20,6 @@ USE_CUDA        = FALSE
 USE_HIP         = FALSE
 
 # PeleLMeX
-USE_TURBINFLOW  = TRUE
 
 # PelePhysics
 Chemistry_Model = air

Exec/RegTests/TurbInflow/input.3d

@@ -29,12 +29,12 @@ prob.P_mean = 101325.0
 prob.flowDir = 2
 prob.flowMag = 5
 
-prob.turb_file = Turb_AJA
-prob.turb_scale_loc = 1.0
-prob.turb_scale_vel = 0.1
-prob.turb_center = 0.005 0.005
-prob.turb_conv_vel = 5.
-prob.turb_nplane = 32
+turbinflow.turb_file = TurbFileHIT/TurbTEST
+turbinflow.turb_scale_loc = 633.151
+turbinflow.turb_scale_vel = 0.5
+turbinflow.turb_center = 0.005 0.005
+turbinflow.turb_conv_vel = 5.
+turbinflow.turb_nplane = 32
 
 #------------  INPUTS TO CONSTANT TRANSPORT -----------------
 transport.const_viscosity        = 0.0001 
@@ -70,6 +70,6 @@ amr.O2.value_greater = 0.4
 amr.O2.field_name    = Y(O2)
 
 fabarray.mfiter_tile_size = 1024 1024 1024
-#amrex.fpe_trap_invalid = 1
-#amrex.fpe_trap_zero = 1
-#amrex.fpe_trap_overflow = 1
+amrex.fpe_trap_invalid = 1
+amrex.fpe_trap_zero = 1
+amrex.fpe_trap_overflow = 1

Exec/RegTests/TurbInflow/input.3d_BoxLoZ

@@ -29,12 +29,12 @@ prob.P_mean = 101325.0
 prob.flowDir = 2
 prob.flowMag = 5
 
-prob.turb_file = TurbFileHIT/TurbTEST
-prob.turb_scale_loc = 633.151
-prob.turb_scale_vel = 0.5
-prob.turb_center = 0.005 0.005
-prob.turb_conv_vel = 5.
-prob.turb_nplane = 32
+turbinflow.turb_file = TurbFileHIT/TurbTEST
+turbinflow.turb_scale_loc = 633.151
+turbinflow.turb_scale_vel = 0.5
+turbinflow.turb_center = 0.005 0.005
+turbinflow.turb_conv_vel = 5.
+turbinflow.turb_nplane = 32
 
 #------------  INPUTS TO CONSTANT TRANSPORT -----------------
 transport.const_viscosity        = 0.0001 
@@ -73,6 +73,6 @@ amr.loBoxZ.in_box_lo = 0.0 0.0 0.0
 amr.loBoxZ.in_box_hi = 0.01 0.01 0.005
 
 fabarray.mfiter_tile_size = 1024 1024 1024
-#amrex.fpe_trap_invalid = 1
-#amrex.fpe_trap_zero = 1
-#amrex.fpe_trap_overflow = 1
+amrex.fpe_trap_invalid = 1
+amrex.fpe_trap_zero = 1
+amrex.fpe_trap_overflow = 1

Exec/RegTests/TurbInflow/input.3d_negY

@@ -29,12 +29,12 @@ prob.P_mean = 101325.0
 prob.flowDir = 1
 prob.flowMag = -5
 
-prob.turb_file = Turb_AJA
-prob.turb_scale_loc = 1.0
-prob.turb_scale_vel = 0.1
-prob.turb_center = 0.005 0.005
-prob.turb_conv_vel = 5.
-prob.turb_nplane = 32
+turbinflow.turb_file = TurbFileHIT/TurbTEST
+turbinflow.turb_scale_loc = 633.151
+turbinflow.turb_scale_vel = 0.5
+turbinflow.turb_center = 0.005 0.005
+turbinflow.turb_conv_vel = 5.
+turbinflow.turb_nplane = 32
 
 #------------  INPUTS TO CONSTANT TRANSPORT -----------------
 transport.const_viscosity        = 0.0001 
@@ -70,6 +70,6 @@ amr.O2.value_greater = 0.4
 amr.O2.field_name    = Y(O2)
 
 fabarray.mfiter_tile_size = 1024 1024 1024
-#amrex.fpe_trap_invalid = 1
-#amrex.fpe_trap_zero = 1
-#amrex.fpe_trap_overflow = 1
+amrex.fpe_trap_invalid = 1
+amrex.fpe_trap_zero = 1
+amrex.fpe_trap_overflow = 1

Exec/RegTests/TurbInflow/input.3d_posX

@@ -29,12 +29,12 @@ prob.P_mean = 101325.0
 prob.flowDir = 0
 prob.flowMag = 5
 
-prob.turb_file = Turb_AJA
-prob.turb_scale_loc = 1.0
-prob.turb_scale_vel = 0.1
-prob.turb_center = 0.005 0.005
-prob.turb_conv_vel = 5.
-prob.turb_nplane = 32
+turbinflow.turb_file = TurbFileHIT/TurbTEST
+turbinflow.turb_scale_loc = 633.151
+turbinflow.turb_scale_vel = 0.5
+turbinflow.turb_center = 0.005 0.005
+turbinflow.turb_conv_vel = 5.
+turbinflow.turb_nplane = 32
 
 #------------  INPUTS TO CONSTANT TRANSPORT -----------------
 transport.const_viscosity        = 0.0001 
@@ -70,6 +70,6 @@ amr.O2.value_greater = 0.4
 amr.O2.field_name    = Y(O2)
 
 fabarray.mfiter_tile_size = 1024 1024 1024
-#amrex.fpe_trap_invalid = 1
-#amrex.fpe_trap_zero = 1
-#amrex.fpe_trap_overflow = 1
+amrex.fpe_trap_invalid = 1
+amrex.fpe_trap_zero = 1
+amrex.fpe_trap_overflow = 1

Exec/RegTests/TurbInflow/pelelm_prob.cpp

@@ -10,58 +10,6 @@ void PeleLM::readProbParm()
     pp.query("flowDir", prob_parm->meanFlowDir);
     pp.query("flowMag", prob_parm->meanFlowMag);
 
-    auto problo = geom[0].ProbLo();
-    auto probhi = geom[0].ProbHi();
-
-    prob_parm->do_turb = false;
-#ifdef PELE_USE_TURBINFLOW
-    if (pp.countval("turb_file") > 0) {
-#if AMREX_SPACEDIM==2
-        amrex::Abort("Turbulence inflow unsupported in 2D.");
-#endif
-        std::string turb_file = ""; 
-        pp.query("turb_file", turb_file);
-        amrex::Real turb_scale_loc = 1.0;
-        pp.query("turb_scale_loc", turb_scale_loc);
-        amrex::Real turb_scale_vel = 1.0;
-        pp.query("turb_scale_vel", turb_scale_vel);
-
-        PeleLM::prob_parm->do_turb = true;
-
-        // Hold nose here - required because of dynamically allocated data in tp
-        AMREX_ASSERT_WITH_MESSAGE(PeleLM::prob_parm->tp.tph == nullptr,"Can only be one TurbParmHost");
-        PeleLM::prob_parm->tp.tph = new TurbParmHost;
-
-        amrex::Vector<amrex::Real> turb_center(2,0.0);
-        if ( prob_parm->meanFlowDir == 0 ) {
-            turb_center[0] = 0.5 * (probhi[1] + problo[1]);
-            turb_center[1] = 0.5 * (probhi[2] + problo[2]);
-        } else if ( prob_parm->meanFlowDir == 1 ) {
-            turb_center[0] = 0.5 * (probhi[0] + problo[0]);
-            turb_center[1] = 0.5 * (probhi[2] + problo[2]);
-        } else if ( prob_parm->meanFlowDir == 2 ) {
-            turb_center[0] = 0.5 * (probhi[0] + problo[0]);
-            turb_center[1] = 0.5 * (probhi[1] + problo[1]);
-        }
-
-        pp.queryarr("turb_center", turb_center);
-        AMREX_ASSERT_WITH_MESSAGE(turb_center.size() == 2, "turb_center must have two elements");
-        for (int n = 0; n < turb_center.size(); ++n) {
-          turb_center[n] *= turb_scale_loc;
-        }
-
-        int turb_nplane = 32; 
-        pp.query("turb_nplane", turb_nplane);
-        AMREX_ASSERT(turb_nplane > 0); 
-        amrex::Real turb_conv_vel = 1;
-        pp.query("turb_conv_vel", turb_conv_vel);
-        AMREX_ASSERT(turb_conv_vel > 0); 
-
-        init_turbinflow(turb_file, turb_scale_loc, turb_scale_vel, turb_center, turb_conv_vel,
-                        turb_nplane, PeleLM::prob_parm->tp);
-   }
-#endif
-
    /*
    if (!m_incompressible) {
       auto& trans_parm = PeleLM::trans_parms.host_trans_parm();

Exec/RegTests/TurbInflow/pelelm_prob_parm.H

@@ -3,9 +3,6 @@
 
 #include <AMReX_REAL.H>
 #include <AMReX_GpuMemory.H>
-#ifdef PELE_USE_TURBINFLOW
-#include <turbinflow.H>
-#endif
 
 using namespace amrex::literals;
 
@@ -15,9 +12,5 @@ struct ProbParm
    amrex::Real P_mean = 101325.0_rt;
    amrex::Real meanFlowMag = 0.0;
    int  meanFlowDir = 1;
-
-   TurbParm tp;
-   amrex::GpuArray<bool,2*AMREX_SPACEDIM> turb_ok = {{false}};
-   bool do_turb = false;
 };
 #endif

Source/PeleLM.H

@@ -12,6 +12,7 @@
 #include <mechanism.H>
 #include "PelePhysics.H"
 #include "PMFData.H"
+#include "turbinflow.H"
 #include "ReactorBase.H"
 #include <PeleLM_Index.H>
 #include <PeleLMDerive.H>
@@ -494,9 +495,7 @@ class PeleLM : public amrex::AmrCore {
    // Fill physical boundaries
    void setInflowBoundaryVel (amrex::MultiFab &a_vel, int lev, PeleLM::TimeStamp a_time);
 
-#ifdef PELE_USE_TURBINFLOW
    void fillTurbInflow(amrex::MultiFab &a_vel, int vel_comp, int lev, const amrex::Real a_time);
-#endif
 
    // Average down operations
    void averageDownState(const PeleLM::TimeStamp &a_time);
@@ -896,6 +895,9 @@ class PeleLM : public amrex::AmrCore {
    std::string m_chem_integrator;
    std::unique_ptr<pele::physics::reactions::ReactorBase> m_reactor;
 
+   // Turbulence injection
+   pele::physics::turbinflow::TurbInflow turb_inflow;
+
    // Linear solvers
    std::unique_ptr<DiffusionOp> m_diffusion_op;
    std::unique_ptr<DiffusionOp> m_mcdiffusion_op;