Particle mkd only (#103)

* Added check for anisotropic meshes, removed unnecessary MF reset for external sources, removed sprayMK functions so only 1 spray update is used per timestep

* Updated PeleMP submodule

* Updated all submodules

* Fix bug with redistribute only happening before time step

* Updated PeleMP submodule

Exec/RegTests/SprayTest/SprayParticlesInitInsert.cpp

@@ -8,8 +8,8 @@ unflatten_particles(const amrex::ULong idx, const amrex::IntVect& max_parts)
   amrex::IntVect indx;
   amrex::ULong cidx = idx;
   amrex::ULong d1 = max_parts[0];
-  amrex::ULong d2 = max_parts[1];
 #if AMREX_SPACEDIM > 2
+  amrex::ULong d2 = max_parts[1];
   indx[2] = int(cidx / (d1 * d2));
   cidx -= amrex::ULong(indx[2]) * d1 * d2;
 #endif

Exec/RegTests/SprayTest/input.2d

@@ -21,12 +21,12 @@ amr.ref_ratio = 2 2 2
 
 amr.blocking_factor = 32               # block factor in grid generation (min box size)
 amr.max_grid_size   = 32               # max box size
-amr.initial_grid_file = two_d_gridfiles/gridfile_32
+amr.initial_grid_file = two_d_gridfiles/gridfile_32_1
 amr.regrid_file = two_d_gridfiles/gridfile_32_2
 
 # amr.blocking_factor = 64               # block factor in grid generation (min box size)
 # amr.max_grid_size   = 64               # max box size
-# amr.initial_grid_file = two_d_gridfiles/gridfile_64
+# amr.initial_grid_file = two_d_gridfiles/gridfile_64_1
 # amr.regrid_file = two_d_gridfiles/gridfile_64_2
 
 #--------------------------- Problem -------------------------------

Exec/RegTests/SprayTest/pelelm_prob.H

@@ -17,17 +17,10 @@ void pelelm_initdata(int i, int j, int k,
                      int /*is_incompressible*/,
                      amrex::Array4<amrex::Real> const& state,
                      amrex::Array4<amrex::Real> const& /*aux*/,
-                     amrex::GeometryData const& geomdata,
+                     amrex::GeometryData const& /*geomdata*/,
                      ProbParm const& prob_parm,
                      pele::physics::PMF::PmfData::DataContainer const * /*pmf_data*/)
 {
-    const amrex::Real* prob_lo = geomdata.ProbLo();
-    const amrex::Real* prob_hi = geomdata.ProbHi();
-    const amrex::Real* dx      = geomdata.CellSize();
-
-    AMREX_D_TERM(const amrex::Real x = prob_lo[0] + (i+0.5)*dx[0];,
-                 const amrex::Real y = prob_lo[1] + (j+0.5)*dx[1];,
-                 const amrex::Real z = prob_lo[2] + (k+0.5)*dx[2];);
 
     auto eos = pele::physics::PhysicsType::eos();
     amrex::Real massfrac[NUM_SPECIES] = {0.0};
@@ -68,17 +61,17 @@ bcnormal(
   const amrex::Real* /*x[AMREX_SPACEDIM]*/,
   const int /*m_nAux*/,
   amrex::Real s_ext[NVAR],
-  const int idir,
-  const int sgn,
+  const int /*idir*/,
+  const int /*sgn*/,
   const amrex::Real /*time*/,
-  amrex::GeometryData const& geomdata,
+  amrex::GeometryData const& /*geomdata*/,
   ProbParm const& prob_parm,
   pele::physics::PMF::PmfData::DataContainer const* /*pmf_data*/)
 {
   AMREX_D_TERM(s_ext[VELX] = prob_parm.vel;, s_ext[VELY] = 0.;
                , s_ext[VELZ] = 0.;);
   amrex::GpuArray<amrex::Real, NUM_SPECIES> massfrac = {{0.0}};
-    massfrac[N2_ID] = prob_parm.Y_N2;
+  massfrac[N2_ID] = prob_parm.Y_N2;
   massfrac[O2_ID] = prob_parm.Y_O2;
   s_ext[TEMP] = prob_parm.T0;
   amrex::Real rho_cgs, P_cgs;

Source/PeleLM.H

@@ -456,12 +456,8 @@ class PeleLM : public amrex::AmrCore {
    void sprayMKDLevel(const int level,
                       const amrex::Real time,
                       const amrex::Real dt);
-   void sprayMK(const amrex::Real time,
-                const amrex::Real dt);
-   void sprayMKLevel(const int level,
-                     const amrex::Real time,
-                     const amrex::Real dt);
-   void sprayInjectRedist(bool regridded, int lbase = 0);
+   void sprayInjectRedist();
+   void sprayPostRegrid();
    void setSprayState(const amrex::Real& a_flow_dt);
    void addSpraySource(const int level);
    static int write_spray_ascii_files;

Source/PeleLMAdvance.cpp

@@ -70,7 +70,6 @@ void PeleLM::Advance(int is_initIter) {
                                     m_nGrowAdv, m_nGrowMAC));
 
    for (int lev = 0; lev <= finest_level; lev++) {
-     m_extSource[lev]->define(grids[lev], dmap[lev], NVAR, amrex::max(m_nGrowAdv, m_nGrowMAC), MFInfo(), *m_factory[lev]);
      m_extSource[lev]->setVal(0.);
    }
    //----------------------------------------------------------------
@@ -182,19 +181,9 @@ void PeleLM::Advance(int is_initIter) {
       averageDownnE(AmrNewTime);
 #endif
       fillPatchState(AmrNewTime);
-      // Reset external sources to zero
-      for (int lev = 0; lev <= finest_level; ++lev) {
-         m_extSource[lev]->setVal(0.);
-      }
 
-#ifdef PELELM_USE_SPRAY
-      if (!is_initIter) {
-         sprayMK(m_cur_time + m_dt, m_dt);
-      }
-#endif
 #ifdef PELELM_USE_SOOT
       if (do_soot_solve) {
-         computeSootSource(AmrNewTime, m_dt);
          clipSootMoments();
       }
 #endif

Source/PeleLMEvolve.cpp

@@ -31,15 +31,21 @@ void PeleLM::Evolve() {
       }
 #ifdef PELELM_USE_SPRAY
       // Inject and redistribute spray particles
-      if (do_spray_particles) {
-        sprayInjectRedist(regridded);
+      if (do_spray_particles && regridded) {
+        sprayPostRegrid();
       }
 #endif
       int is_init = 0;
       Advance(is_init);
       m_nstep++;
       m_cur_time += m_dt;
 
+#ifdef PELELM_USE_SPRAY
+      if (do_spray_particles) {
+        sprayInjectRedist();
+      }
+#endif
+
       // Temporals
       if (doTemporalsNow()) {
          writeTemporals();

Source/PeleLMSetup.cpp

@@ -23,6 +23,11 @@ static Box grow_box_by_two (const Box& b) { return amrex::grow(b,2); }
 void PeleLM::Setup() {
    BL_PROFILE("PeleLM::Setup()");
 
+   // Ensure grid is isotropic
+   {
+     auto const dx = geom[0].CellSizeArray();
+     AMREX_ALWAYS_ASSERT(AMREX_D_TERM(,dx[0] == dx[1], && dx[1] == dx[2]));
+   }
    // Print build info to screen
    const char* githash1 = buildInfoGetGitHash(1);
    const char* githash2 = buildInfoGetGitHash(2);

Source/Spray/PeleLMSprayParticles.cpp

@@ -32,6 +32,8 @@ Vector<Real> spray_cfl;
 Vector<int> spray_state_ghosts;
 Vector<int> spray_source_ghosts;
 Vector<int> spray_ghost_num;
+Vector<int> prev_state;
+Vector<int> prev_source;
 
 void
 RemoveParticlesOnExit()
@@ -43,6 +45,7 @@ RemoveParticlesOnExit()
   delete VirtPC;
   VirtPC = nullptr;
 }
+bool mesh_regrid = true;
 std::string init_file;
 int init_function = 1;
 int spray_verbose = 0;
@@ -105,11 +108,12 @@ PeleLM::readSprayParameters()
   ParmParse pp("peleLM");
 
   pp.query("do_spray_particles", do_spray_particles);
-  if (do_spray_particles != 1) {
-    do_spray_particles = 0;
+  if (!do_spray_particles) {
     return;
   }
   const Real temp_cfl = max_spray_cfl;
+  // Mush change dtmod to 1 since we only do MKD
+  sprayData.dtmod = 1.;
   SprayParticleContainer::readSprayParams(
     spray_verbose, max_spray_cfl, wall_temp, mass_trans, mom_trans,
     write_spray_ascii_files, plot_spray_src, init_function, init_file,
@@ -251,7 +255,8 @@ PeleLM::initSprays()
       Abort("Must initialize spray particles with particles.init_function or "
             "particles.init_file");
     }
-    sprayInjectRedist(true, 0);
+    sprayPostRegrid();
+    sprayInjectRedist();
     if (spray_verbose >= 1) {
       amrex::Print() << "Total number of initial particles "
                      << theSprayPC()->TotalNumberOfParticles(false, false)
@@ -273,6 +278,7 @@ PeleLM::sprayRestart()
     amrex::ExecOnFinalize(RemoveParticlesOnExit);
     {
       theSprayPC()->Restart(m_restart_chkfile, "particles", true);
+      sprayPostRegrid();
       amrex::Gpu::Device::streamSynchronize();
     }
   }
@@ -324,14 +330,19 @@ PeleLM::setSprayState(const Real& a_flow_dt)
       lev, finest_level, 1, spray_cfl[lev]);
     spray_state_ghosts[lev] = state_ghosts;
     spray_source_ghosts[lev] = source_ghosts;
-    m_spraystate[lev].reset(new MultiFab(
-      grids[lev], dmap[lev], NVAR, state_ghosts, MFInfo(), *m_factory[lev]));
+    if (
+      mesh_regrid || prev_state[lev] != state_ghosts ||
+      prev_source[lev] != source_ghosts) {
+      m_spraystate[lev].reset(new MultiFab(
+        grids[lev], dmap[lev], NVAR, state_ghosts, MFInfo(), *m_factory[lev]));
+      m_spraysource[lev].reset(new MultiFab(
+        grids[lev], dmap[lev], num_spray_src, source_ghosts, MFInfo(),
+        *m_factory[lev]));
+    }
     fillpatch_state(lev, m_cur_time, *(m_spraystate[lev].get()), state_ghosts);
-    m_spraysource[lev].reset(new MultiFab(
-      grids[lev], dmap[lev], num_spray_src, source_ghosts, MFInfo(),
-      *m_factory[lev]));
     m_spraysource[lev]->setVal(0.);
   }
+  mesh_regrid = false;
 }
 
 void
@@ -376,7 +387,6 @@ PeleLM::sprayMKD(const Real time, const Real dt)
     removeVirtualParticles(lev);
     m_spraysource[lev]->setVal(0.);
   }
-  theSprayPC()->Redistribute();
 }
 
 void
@@ -422,110 +432,67 @@ PeleLM::sprayMKDLevel(const int level, const Real time, const Real dt)
 }
 
 void
-PeleLM::sprayMK(const Real time, const Real dt)
+PeleLM::sprayPostRegrid()
 {
-  if (!do_spray_particles) {
-    return;
-  }
-  setupVirtualParticles(0);
-  for (int lev = 0; lev <= finest_level; ++lev) {
-    if (spray_verbose > 1) {
-      amrex::Print() << "sprayMKLevel " << lev << std::endl;
+  static Vector<BoxArray> ba_spray;
+  static Vector<DistributionMapping> dm_spray;
+  bool changed = false;
+  if (ba_spray.size() != finest_level + 1) {
+    ba_spray.resize(finest_level + 1);
+    dm_spray.resize(finest_level + 1);
+    prev_state.resize(finest_level + 1);
+    prev_source.resize(finest_level + 1);
+    changed = true;
+  } else {
+    for (int lev = 0; lev <= finest_level && !changed; lev++) {
+      if (ba_spray[lev] != grids[lev]) {
+        changed = true;
+      }
+      if (!changed && dm_spray[lev] != dmap[lev]) {
+        changed = true;
+      }
     }
-    sprayMKLevel(lev, time, dt);
-    addSpraySource(lev);
-    removeVirtualParticles(lev);
-    removeGhostParticles(lev);
-  }
-}
-
-void
-PeleLM::sprayMKLevel(const int level, const Real time, const Real dt)
-{
-  auto ldata_p = getLevelDataPtr(level, AmrNewTime);
-  amrex::MultiFab& source = *(m_spraysource[level].get());
-  amrex::MultiFab& state = ldata_p->state;
-  auto const* ltransparm = PeleLM::trans_parms.device_trans_parm();
-  int state_ghosts = 2;
-  int source_ghosts = 2;
-  if (level < finest_level) {
-    setupGhostParticles(1, level);
-  }
-  bool isVirt = false;
-  bool isGhost = false;
-  theSprayPC()->moveKick(
-    state, source, level, dt, time, isVirt, isGhost, state_ghosts,
-    source_ghosts, ltransparm);
-  if (level < finest_level && theVirtPC() != nullptr) {
-    isVirt = true;
-    isGhost = false;
-    theVirtPC()->moveKick(
-      state, source, level, dt, time, isVirt, isGhost, state_ghosts,
-      source_ghosts, ltransparm);
   }
-  if (theGhostPC() != nullptr && level != 0) {
-    isVirt = false;
-    isGhost = true;
-    theGhostPC()->moveKick(
-      state, source, level, dt, time, isVirt, isGhost, state_ghosts,
-      source_ghosts, ltransparm);
+  // Update the local BoxArray and DistributionMap
+  if (changed) {
+    mesh_regrid = true;
+    for (int lev = 0; lev <= finest_level; ++lev) {
+      ba_spray[lev] = grids[lev];
+      dm_spray[lev] = dmap[lev];
+      prev_state[lev] = -1;
+      prev_source[lev] = -1;
+    }
+    theSprayPC()->Redistribute();
   }
-  source.SumBoundary(geom[level].periodicity());
 }
 
 void
-PeleLM::sprayInjectRedist(bool regridded, int lbase)
+PeleLM::sprayInjectRedist()
 {
-  if (lbase > 0) {
-    return;
-  }
   BL_PROFILE("PeleLM::sprayInjectRedist");
-  if (theSprayPC()) {
-    bool injected = false;
-    static Vector<BoxArray> ba_spray;
-    static Vector<DistributionMapping> dm_spray;
-    for (int lev = 0; lev <= finest_level; ++lev) {
-      int nstep = 0; // Unused
-      BL_PROFILE_VAR("SprayParticles::injectParticles()", INJECT_SPRAY);
-      ProbParm const* lprobparm = prob_parm;
-      Real cur_time = m_t_new[lev]; // Still the time from the last time step
-      Real dt = m_dt;
-      bool lev_injected = theSprayPC()->injectParticles(
-        cur_time, dt, nstep, lev, finest_level, *lprobparm);
-      if (!injected && lev_injected)
-        injected = true;
-      BL_PROFILE_VAR_STOP(INJECT_SPRAY);
-    }
-    bool changed = false;
-    if (regridded) {
-      if (ba_spray.size() != finest_level + 1) {
-        ba_spray.resize(finest_level + 1);
-        dm_spray.resize(finest_level + 1);
-        changed = true;
-      } else {
-        for (int lev = 0; lev <= finest_level && !changed; lev++) {
-          if (ba_spray[lev] != grids[lev]) {
-            changed = true;
-          }
-          if (!changed && dm_spray[lev] != dmap[lev]) {
-            changed = true;
-          }
-        }
-      }
-    }
-    // Update the local BoxArray and DistributionMap
-    if (changed) {
-      for (int lev = 0; lev <= finest_level; ++lev) {
-        ba_spray[lev] = grids[lev];
-        dm_spray[lev] = dmap[lev];
-      }
-    }
-    // We redistributed after the MKD, only need to redistribute
-    // if injection or regrid occurs
-    if (changed || injected) {
-      theSprayPC()->Redistribute();
+  Long prev_count = 0;
+  if (spray_verbose >= 3) {
+    prev_count = theSprayPC()->TotalNumberOfParticles(true, false);
+  }
+  bool injected = false;
+  for (int lev = 0; lev <= finest_level; ++lev) {
+    int nstep = 0; // Unused
+    ProbParm const* lprobparm = prob_parm;
+    Real cur_time = m_t_new[lev]; // Still the time from the last time step
+    Real dt = m_dt;
+    bool lev_injected = theSprayPC()->injectParticles(
+      cur_time, dt, nstep, lev, finest_level, *lprobparm);
+    if (lev_injected) {
+      injected = true;
     }
   }
+  // We must redistribute after each time step
+  theSprayPC()->Redistribute();
+  if (spray_verbose >= 3 && injected) {
+    Long new_count = theSprayPC()->TotalNumberOfParticles(true, false);
+    Long num_inj = new_count - prev_count;
+    amrex::Print() << "Injected " << num_inj << " particles at time " << m_t_new[0] << std::endl;
+  }
 }
 
 #endif