From b4493597cd35f46b9bb0126ca6017977d6c7a3ed Mon Sep 17 00:00:00 2001 From: ldowen <54121008+ldowen@users.noreply.github.com> Date: Thu, 30 Jun 2022 11:14:32 -0700 Subject: [PATCH] Particle mkd only (#103) * Added check for anisotropic meshes, removed unnecessary MF reset for external sources, removed sprayMK functions so only 1 spray update is used per timestep * Updated PeleMP submodule * Updated all submodules * Fix bug with redistribute only happening before time step * Updated PeleMP submodule --- .../SprayTest/SprayParticlesInitInsert.cpp | 2 +- Exec/RegTests/SprayTest/input.2d | 4 +- Exec/RegTests/SprayTest/pelelm_prob.H | 17 +- Source/PeleLM.H | 8 +- Source/PeleLMAdvance.cpp | 11 -- Source/PeleLMEvolve.cpp | 10 +- Source/PeleLMSetup.cpp | 5 + Source/Spray/PeleLMSprayParticles.cpp | 171 +++++++----------- Submodules/PeleMP | 2 +- Submodules/amrex | 2 +- 10 files changed, 94 insertions(+), 138 deletions(-) diff --git a/Exec/RegTests/SprayTest/SprayParticlesInitInsert.cpp b/Exec/RegTests/SprayTest/SprayParticlesInitInsert.cpp index 3f8427db..fca50fe9 100644 --- a/Exec/RegTests/SprayTest/SprayParticlesInitInsert.cpp +++ b/Exec/RegTests/SprayTest/SprayParticlesInitInsert.cpp @@ -8,8 +8,8 @@ unflatten_particles(const amrex::ULong idx, const amrex::IntVect& max_parts) amrex::IntVect indx; amrex::ULong cidx = idx; amrex::ULong d1 = max_parts[0]; - amrex::ULong d2 = max_parts[1]; #if AMREX_SPACEDIM > 2 + amrex::ULong d2 = max_parts[1]; indx[2] = int(cidx / (d1 * d2)); cidx -= amrex::ULong(indx[2]) * d1 * d2; #endif diff --git a/Exec/RegTests/SprayTest/input.2d b/Exec/RegTests/SprayTest/input.2d index 45181927..ca28a993 100644 --- a/Exec/RegTests/SprayTest/input.2d +++ b/Exec/RegTests/SprayTest/input.2d @@ -21,12 +21,12 @@ amr.ref_ratio = 2 2 2 amr.blocking_factor = 32 # block factor in grid generation (min box size) amr.max_grid_size = 32 # max box size -amr.initial_grid_file = two_d_gridfiles/gridfile_32 +amr.initial_grid_file = two_d_gridfiles/gridfile_32_1 amr.regrid_file = two_d_gridfiles/gridfile_32_2 # amr.blocking_factor = 64 # block factor in grid generation (min box size) # amr.max_grid_size = 64 # max box size -# amr.initial_grid_file = two_d_gridfiles/gridfile_64 +# amr.initial_grid_file = two_d_gridfiles/gridfile_64_1 # amr.regrid_file = two_d_gridfiles/gridfile_64_2 #--------------------------- Problem ------------------------------- diff --git a/Exec/RegTests/SprayTest/pelelm_prob.H b/Exec/RegTests/SprayTest/pelelm_prob.H index 87c81518..661fe500 100644 --- a/Exec/RegTests/SprayTest/pelelm_prob.H +++ b/Exec/RegTests/SprayTest/pelelm_prob.H @@ -17,17 +17,10 @@ void pelelm_initdata(int i, int j, int k, int /*is_incompressible*/, amrex::Array4 const& state, amrex::Array4 const& /*aux*/, - amrex::GeometryData const& geomdata, + amrex::GeometryData const& /*geomdata*/, ProbParm const& prob_parm, pele::physics::PMF::PmfData::DataContainer const * /*pmf_data*/) { - const amrex::Real* prob_lo = geomdata.ProbLo(); - const amrex::Real* prob_hi = geomdata.ProbHi(); - const amrex::Real* dx = geomdata.CellSize(); - - AMREX_D_TERM(const amrex::Real x = prob_lo[0] + (i+0.5)*dx[0];, - const amrex::Real y = prob_lo[1] + (j+0.5)*dx[1];, - const amrex::Real z = prob_lo[2] + (k+0.5)*dx[2];); auto eos = pele::physics::PhysicsType::eos(); amrex::Real massfrac[NUM_SPECIES] = {0.0}; @@ -68,17 +61,17 @@ bcnormal( const amrex::Real* /*x[AMREX_SPACEDIM]*/, const int /*m_nAux*/, amrex::Real s_ext[NVAR], - const int idir, - const int sgn, + const int /*idir*/, + const int /*sgn*/, const amrex::Real /*time*/, - amrex::GeometryData const& geomdata, + amrex::GeometryData const& /*geomdata*/, ProbParm const& prob_parm, pele::physics::PMF::PmfData::DataContainer const* /*pmf_data*/) { AMREX_D_TERM(s_ext[VELX] = prob_parm.vel;, s_ext[VELY] = 0.; , s_ext[VELZ] = 0.;); amrex::GpuArray massfrac = {{0.0}}; - massfrac[N2_ID] = prob_parm.Y_N2; + massfrac[N2_ID] = prob_parm.Y_N2; massfrac[O2_ID] = prob_parm.Y_O2; s_ext[TEMP] = prob_parm.T0; amrex::Real rho_cgs, P_cgs; diff --git a/Source/PeleLM.H b/Source/PeleLM.H index 8aee2922..69462b4d 100644 --- a/Source/PeleLM.H +++ b/Source/PeleLM.H @@ -456,12 +456,8 @@ class PeleLM : public amrex::AmrCore { void sprayMKDLevel(const int level, const amrex::Real time, const amrex::Real dt); - void sprayMK(const amrex::Real time, - const amrex::Real dt); - void sprayMKLevel(const int level, - const amrex::Real time, - const amrex::Real dt); - void sprayInjectRedist(bool regridded, int lbase = 0); + void sprayInjectRedist(); + void sprayPostRegrid(); void setSprayState(const amrex::Real& a_flow_dt); void addSpraySource(const int level); static int write_spray_ascii_files; diff --git a/Source/PeleLMAdvance.cpp b/Source/PeleLMAdvance.cpp index b160b9b9..cd4c0e6b 100644 --- a/Source/PeleLMAdvance.cpp +++ b/Source/PeleLMAdvance.cpp @@ -70,7 +70,6 @@ void PeleLM::Advance(int is_initIter) { m_nGrowAdv, m_nGrowMAC)); for (int lev = 0; lev <= finest_level; lev++) { - m_extSource[lev]->define(grids[lev], dmap[lev], NVAR, amrex::max(m_nGrowAdv, m_nGrowMAC), MFInfo(), *m_factory[lev]); m_extSource[lev]->setVal(0.); } //---------------------------------------------------------------- @@ -182,19 +181,9 @@ void PeleLM::Advance(int is_initIter) { averageDownnE(AmrNewTime); #endif fillPatchState(AmrNewTime); - // Reset external sources to zero - for (int lev = 0; lev <= finest_level; ++lev) { - m_extSource[lev]->setVal(0.); - } -#ifdef PELELM_USE_SPRAY - if (!is_initIter) { - sprayMK(m_cur_time + m_dt, m_dt); - } -#endif #ifdef PELELM_USE_SOOT if (do_soot_solve) { - computeSootSource(AmrNewTime, m_dt); clipSootMoments(); } #endif diff --git a/Source/PeleLMEvolve.cpp b/Source/PeleLMEvolve.cpp index af587c96..e52c4559 100644 --- a/Source/PeleLMEvolve.cpp +++ b/Source/PeleLMEvolve.cpp @@ -31,8 +31,8 @@ void PeleLM::Evolve() { } #ifdef PELELM_USE_SPRAY // Inject and redistribute spray particles - if (do_spray_particles) { - sprayInjectRedist(regridded); + if (do_spray_particles && regridded) { + sprayPostRegrid(); } #endif int is_init = 0; @@ -40,6 +40,12 @@ void PeleLM::Evolve() { m_nstep++; m_cur_time += m_dt; +#ifdef PELELM_USE_SPRAY + if (do_spray_particles) { + sprayInjectRedist(); + } +#endif + // Temporals if (doTemporalsNow()) { writeTemporals(); diff --git a/Source/PeleLMSetup.cpp b/Source/PeleLMSetup.cpp index 373b9591..a2e0156d 100644 --- a/Source/PeleLMSetup.cpp +++ b/Source/PeleLMSetup.cpp @@ -23,6 +23,11 @@ static Box grow_box_by_two (const Box& b) { return amrex::grow(b,2); } void PeleLM::Setup() { BL_PROFILE("PeleLM::Setup()"); + // Ensure grid is isotropic + { + auto const dx = geom[0].CellSizeArray(); + AMREX_ALWAYS_ASSERT(AMREX_D_TERM(,dx[0] == dx[1], && dx[1] == dx[2])); + } // Print build info to screen const char* githash1 = buildInfoGetGitHash(1); const char* githash2 = buildInfoGetGitHash(2); diff --git a/Source/Spray/PeleLMSprayParticles.cpp b/Source/Spray/PeleLMSprayParticles.cpp index ea00f3b6..7a895414 100644 --- a/Source/Spray/PeleLMSprayParticles.cpp +++ b/Source/Spray/PeleLMSprayParticles.cpp @@ -32,6 +32,8 @@ Vector spray_cfl; Vector spray_state_ghosts; Vector spray_source_ghosts; Vector spray_ghost_num; +Vector prev_state; +Vector prev_source; void RemoveParticlesOnExit() @@ -43,6 +45,7 @@ RemoveParticlesOnExit() delete VirtPC; VirtPC = nullptr; } +bool mesh_regrid = true; std::string init_file; int init_function = 1; int spray_verbose = 0; @@ -105,11 +108,12 @@ PeleLM::readSprayParameters() ParmParse pp("peleLM"); pp.query("do_spray_particles", do_spray_particles); - if (do_spray_particles != 1) { - do_spray_particles = 0; + if (!do_spray_particles) { return; } const Real temp_cfl = max_spray_cfl; + // Mush change dtmod to 1 since we only do MKD + sprayData.dtmod = 1.; SprayParticleContainer::readSprayParams( spray_verbose, max_spray_cfl, wall_temp, mass_trans, mom_trans, write_spray_ascii_files, plot_spray_src, init_function, init_file, @@ -251,7 +255,8 @@ PeleLM::initSprays() Abort("Must initialize spray particles with particles.init_function or " "particles.init_file"); } - sprayInjectRedist(true, 0); + sprayPostRegrid(); + sprayInjectRedist(); if (spray_verbose >= 1) { amrex::Print() << "Total number of initial particles " << theSprayPC()->TotalNumberOfParticles(false, false) @@ -273,6 +278,7 @@ PeleLM::sprayRestart() amrex::ExecOnFinalize(RemoveParticlesOnExit); { theSprayPC()->Restart(m_restart_chkfile, "particles", true); + sprayPostRegrid(); amrex::Gpu::Device::streamSynchronize(); } } @@ -324,14 +330,19 @@ PeleLM::setSprayState(const Real& a_flow_dt) lev, finest_level, 1, spray_cfl[lev]); spray_state_ghosts[lev] = state_ghosts; spray_source_ghosts[lev] = source_ghosts; - m_spraystate[lev].reset(new MultiFab( - grids[lev], dmap[lev], NVAR, state_ghosts, MFInfo(), *m_factory[lev])); + if ( + mesh_regrid || prev_state[lev] != state_ghosts || + prev_source[lev] != source_ghosts) { + m_spraystate[lev].reset(new MultiFab( + grids[lev], dmap[lev], NVAR, state_ghosts, MFInfo(), *m_factory[lev])); + m_spraysource[lev].reset(new MultiFab( + grids[lev], dmap[lev], num_spray_src, source_ghosts, MFInfo(), + *m_factory[lev])); + } fillpatch_state(lev, m_cur_time, *(m_spraystate[lev].get()), state_ghosts); - m_spraysource[lev].reset(new MultiFab( - grids[lev], dmap[lev], num_spray_src, source_ghosts, MFInfo(), - *m_factory[lev])); m_spraysource[lev]->setVal(0.); } + mesh_regrid = false; } void @@ -376,7 +387,6 @@ PeleLM::sprayMKD(const Real time, const Real dt) removeVirtualParticles(lev); m_spraysource[lev]->setVal(0.); } - theSprayPC()->Redistribute(); } void @@ -422,110 +432,67 @@ PeleLM::sprayMKDLevel(const int level, const Real time, const Real dt) } void -PeleLM::sprayMK(const Real time, const Real dt) +PeleLM::sprayPostRegrid() { - if (!do_spray_particles) { - return; - } - setupVirtualParticles(0); - for (int lev = 0; lev <= finest_level; ++lev) { - if (spray_verbose > 1) { - amrex::Print() << "sprayMKLevel " << lev << std::endl; + static Vector ba_spray; + static Vector dm_spray; + bool changed = false; + if (ba_spray.size() != finest_level + 1) { + ba_spray.resize(finest_level + 1); + dm_spray.resize(finest_level + 1); + prev_state.resize(finest_level + 1); + prev_source.resize(finest_level + 1); + changed = true; + } else { + for (int lev = 0; lev <= finest_level && !changed; lev++) { + if (ba_spray[lev] != grids[lev]) { + changed = true; + } + if (!changed && dm_spray[lev] != dmap[lev]) { + changed = true; + } } - sprayMKLevel(lev, time, dt); - addSpraySource(lev); - removeVirtualParticles(lev); - removeGhostParticles(lev); - } -} - -void -PeleLM::sprayMKLevel(const int level, const Real time, const Real dt) -{ - auto ldata_p = getLevelDataPtr(level, AmrNewTime); - amrex::MultiFab& source = *(m_spraysource[level].get()); - amrex::MultiFab& state = ldata_p->state; - auto const* ltransparm = PeleLM::trans_parms.device_trans_parm(); - int state_ghosts = 2; - int source_ghosts = 2; - if (level < finest_level) { - setupGhostParticles(1, level); - } - bool isVirt = false; - bool isGhost = false; - theSprayPC()->moveKick( - state, source, level, dt, time, isVirt, isGhost, state_ghosts, - source_ghosts, ltransparm); - if (level < finest_level && theVirtPC() != nullptr) { - isVirt = true; - isGhost = false; - theVirtPC()->moveKick( - state, source, level, dt, time, isVirt, isGhost, state_ghosts, - source_ghosts, ltransparm); } - if (theGhostPC() != nullptr && level != 0) { - isVirt = false; - isGhost = true; - theGhostPC()->moveKick( - state, source, level, dt, time, isVirt, isGhost, state_ghosts, - source_ghosts, ltransparm); + // Update the local BoxArray and DistributionMap + if (changed) { + mesh_regrid = true; + for (int lev = 0; lev <= finest_level; ++lev) { + ba_spray[lev] = grids[lev]; + dm_spray[lev] = dmap[lev]; + prev_state[lev] = -1; + prev_source[lev] = -1; + } + theSprayPC()->Redistribute(); } - source.SumBoundary(geom[level].periodicity()); } void -PeleLM::sprayInjectRedist(bool regridded, int lbase) +PeleLM::sprayInjectRedist() { - if (lbase > 0) { - return; - } BL_PROFILE("PeleLM::sprayInjectRedist"); - if (theSprayPC()) { - bool injected = false; - static Vector ba_spray; - static Vector dm_spray; - for (int lev = 0; lev <= finest_level; ++lev) { - int nstep = 0; // Unused - BL_PROFILE_VAR("SprayParticles::injectParticles()", INJECT_SPRAY); - ProbParm const* lprobparm = prob_parm; - Real cur_time = m_t_new[lev]; // Still the time from the last time step - Real dt = m_dt; - bool lev_injected = theSprayPC()->injectParticles( - cur_time, dt, nstep, lev, finest_level, *lprobparm); - if (!injected && lev_injected) - injected = true; - BL_PROFILE_VAR_STOP(INJECT_SPRAY); - } - bool changed = false; - if (regridded) { - if (ba_spray.size() != finest_level + 1) { - ba_spray.resize(finest_level + 1); - dm_spray.resize(finest_level + 1); - changed = true; - } else { - for (int lev = 0; lev <= finest_level && !changed; lev++) { - if (ba_spray[lev] != grids[lev]) { - changed = true; - } - if (!changed && dm_spray[lev] != dmap[lev]) { - changed = true; - } - } - } - } - // Update the local BoxArray and DistributionMap - if (changed) { - for (int lev = 0; lev <= finest_level; ++lev) { - ba_spray[lev] = grids[lev]; - dm_spray[lev] = dmap[lev]; - } - } - // We redistributed after the MKD, only need to redistribute - // if injection or regrid occurs - if (changed || injected) { - theSprayPC()->Redistribute(); + Long prev_count = 0; + if (spray_verbose >= 3) { + prev_count = theSprayPC()->TotalNumberOfParticles(true, false); + } + bool injected = false; + for (int lev = 0; lev <= finest_level; ++lev) { + int nstep = 0; // Unused + ProbParm const* lprobparm = prob_parm; + Real cur_time = m_t_new[lev]; // Still the time from the last time step + Real dt = m_dt; + bool lev_injected = theSprayPC()->injectParticles( + cur_time, dt, nstep, lev, finest_level, *lprobparm); + if (lev_injected) { + injected = true; } } + // We must redistribute after each time step + theSprayPC()->Redistribute(); + if (spray_verbose >= 3 && injected) { + Long new_count = theSprayPC()->TotalNumberOfParticles(true, false); + Long num_inj = new_count - prev_count; + amrex::Print() << "Injected " << num_inj << " particles at time " << m_t_new[0] << std::endl; + } } #endif diff --git a/Submodules/PeleMP b/Submodules/PeleMP index db1c8578..ede2b9a6 160000 --- a/Submodules/PeleMP +++ b/Submodules/PeleMP @@ -1 +1 @@ -Subproject commit db1c85781635783a2bb6dea5340ca92855ef06f6 +Subproject commit ede2b9a63320f4df5ee673ba5b04faa7c5545f4d diff --git a/Submodules/amrex b/Submodules/amrex index b2b9150a..8fb23ec1 160000 --- a/Submodules/amrex +++ b/Submodules/amrex @@ -1 +1 @@ -Subproject commit b2b9150ada12af878a07e0628be03668a9d17270 +Subproject commit 8fb23ec17a58284af6bdafbcda3eea0d86d8ce69