diff --git a/Source/PeleLM.H b/Source/PeleLM.H index 2041d364..f3f9828b 100644 --- a/Source/PeleLM.H +++ b/Source/PeleLM.H @@ -149,7 +149,9 @@ class PeleLM : public amrex::AmrCore { //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- - // Gather State Data of a level into a struct + /** + * \brief Data container for the state data and long lived derived data + */ struct LevelData { LevelData () = default; LevelData (amrex::BoxArray const& ba, @@ -179,6 +181,9 @@ class PeleLM : public amrex::AmrCore { #endif }; + /** + * \brief Data container for reaction data + */ struct LevelDataReact { LevelDataReact () = default; LevelDataReact (const amrex::BoxArray &ba, @@ -193,7 +198,9 @@ class PeleLM : public amrex::AmrCore { //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- - // Gather Advance Datas into a structs + /** + * \brief Advance function data container for diffusion term + */ struct AdvanceDiffData { AdvanceDiffData () = default; AdvanceDiffData (int a_finestLevel, @@ -212,6 +219,9 @@ class PeleLM : public amrex::AmrCore { amrex::Vector DT; // Diffusion term of T (soret flux) amrex::Vector> soret_fluxes; // Soret flux correction }; + /** + * \brief Advance function data container for advection term + */ struct AdvanceAdvData { AdvanceAdvData () = default; AdvanceAdvData (int a_finestLevel, @@ -236,13 +246,35 @@ class PeleLM : public amrex::AmrCore { //----------------------------------------------------------------------------- // INIT - // initData + /** + * \brief Top level initialization function + */ void initData(); - // initLevelData + /** + * \brief Fill the initial level data container + * using the user defined init function + */ void initLevelData(int lev); - // initialIterations + /** + * \brief Fill the initial level data container + * using the data from a pltfile + * \param a_pltFile path to PeleLMeX plot file + */ + void initLevelDataFromPlt(int a_lev, const std::string &a_pltFile); + + /** + * \brief Project the initial solution velocity field, + * initializing the reaction term and providing a first + * estimate of the time step size + */ + void projectInitSolution(); + + /** + * \brief Iterate the advance function to provide an + * initial pressure field. + */ void initialIterations(); // initFromGridFile @@ -253,29 +285,81 @@ class PeleLM : public amrex::AmrCore { amrex::Real time, bool initial); - // checkRunParams + /** + * \brief Check the consistency of the run + * parameters. + */ void checkRunParams(); + //----------------------------------------------------------------------------- + // REGRID / LOAD BALANCE + + /** + * \brief Compute load balancing cost estimate on a given level + * filling the PeleLM class LayoutData with default method + * \param a_lev target level + */ void computeCosts(int a_lev); + + /** + * \brief Compute load balancing cost estimate on a given level + * \param a_lev target level + * \param a_costs LayoutData holding the cost for each box + * \param a_costMethod cost method + */ void computeCosts(int a_lev, amrex::LayoutData &a_costs, int a_costMethod); + /** + * \brief Create/update the DMap used for chemistry on all levels + */ + void loadBalanceChem(); + + /** + * \brief Create/update the DMap used for chemistry on a given level + * \param a_lev level of interest + */ + void loadBalanceChemLev(int a_lev); + //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- - // PROJECTION + // NODAL PROJECTION - // Initial projections + /** + * \brief Initial velocity projection. The pressure and + * pressure gradient are reset to zero after the projection. + * Sigma is 1/rho instead of dt/rho. + */ void initialProjection(); + /** + * \brief Initial velocity projection considering zero + * velocity divergence (no cell-centered RHS) to establish + * the isostatic pressure when using gravity + * Sigma is 1/rho instead of dt/rho. + */ void initialPressProjection(); - // Velocity projection at Time + /** + * \brief Advance function velocity projection. + * \param is_init flag initial iteration calls + * \param a_rhoTime time used to fill sigma + * \param a_dt time time step size + */ void velocityProjection(int is_init, const PeleLM::TimeStamp &a_rhoTime, const amrex::Real &a_dt); - // Actual projection function + /** + * \brief Actual nodal projection call function. + * \param a_vel vector of velocity MultiFabs + * \param a_sigma vector of variable coefficient + * \param rhs_cc vector of cell-centered projection RHS (can be empty) + * \param rhs_cc vector of node-centered projection RHS (can be empty) + * \param increment_gp flag incremental projection (where vel is U^{np1*} - U^{n}) + * \param scaling_factor used for constant coefficient projection + */ void doNodalProject(const amrex::Vector &a_vel, const amrex::Vector &a_sigma, const amrex::Vector &rhs_cc, @@ -283,35 +367,82 @@ class PeleLM : public amrex::AmrCore { int increment_gp, amrex::Real scaling_factor); - // Scaling functions for 2D-RZ calculations + /** + * \brief For 2D-RZ, scale multifab components by radius + * including 1 ghost cell on Dirichlet BC + * \param a_lev level index + * \param a_mf MultiFab to act upon + */ void scaleProj_RZ(int a_lev, - amrex::MultiFab &a_rhs); + amrex::MultiFab &a_mf); + + /** + * \brief For 2D-RZ, unscale multifab components by radius + * \param a_lev level index + * \param a_mf MultiFab to act upon + */ void unscaleProj_RZ(int a_lev, - amrex::MultiFab &a_rhs); + amrex::MultiFab &a_mf); //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- // MAC PROJECTION + /** + * \brief Re-create the MAC projector (reset the std::unique_ptr) + * if the circonstances require it + */ void resetMacProjector(); - // Predict face velocity using Godunov + /** + * \brief Use Godunov to obtain face-centered velocities + * from cell-centered ones + * \param advData container for the face-centered velocity umac + */ void predictVelocity(std::unique_ptr &advData); - // Create the MAC projection RHS + /** + * \brief Create MAC projection RHS by fillpatching divU + * between old and new time + * \param advData container for mac_divu + */ void createMACRHS(std::unique_ptr &advData); - // Append the chi increment + /** + * \brief Create MAC projection RHS by fillpatching divU + * between old and new time + * \param a_sdcIter current SDC iteration index + * \param a_time time stamp to access thermodynamic pressure + * \param advData container for chi and incremented mac_divu + */ void addChiIncrement(int a_sdcIter, const PeleLM::TimeStamp &a_time, std::unique_ptr &advData); - // Project face velocity + /** + * \brief Perform MAC projection to obtained divergence constrained + * face-centered (MAC) velocities + * \param a_time time stamp to access the density used in the projection + * \param advData container for the face-centered velocities umac + * \param a_divu velocity divergence constraint (can be empty) + */ void macProject(const PeleLM::TimeStamp &a_time, std::unique_ptr &advData, const amrex::Vector &a_divu); - // Umac ghost cells + /** + * \brief Fill the projected face velocities ghost faces, ensuring + * that the ghost cell velocities are constrained by the ghost cell + * divergence constraint + * \param a_lev target level + * \param a_nGrow number of ghost faces + * \param crse_geom pointer to the geometry of the next coarser level + * \param fine_geom pointer to the geometry of the current level + * \param u_mac_crse face-centered umac of the next coarser level + * \param u_mac_fine face-centered umac of the current level (those updated by the function) + * \param divu pointer to the current level divU + * \param crse_ratio refinement ratio with the next coarser level + */ void create_constrained_umac_grown(int a_lev, int a_nGrow, const amrex::Geometry* crse_geom, const amrex::Geometry* fine_geom, @@ -320,7 +451,11 @@ class PeleLM : public amrex::AmrCore { const amrex::MultiFab *divu, const amrex::IntVect& crse_ratio); - // MAC proj BC + /** + * \brief Setup the linear operator boundary condition for the + * MAC projection in a given direction + * \param a_side an AMReX orientation object + */ amrex::Array getMACProjectionBC(amrex::Orientation::Side a_side); //----------------------------------------------------------------------------- @@ -328,38 +463,100 @@ class PeleLM : public amrex::AmrCore { //----------------------------------------------------------------------------- // DIFFUSION - // compute cell-centered diffusivity (stored in LevelData) + /** + * \brief Compute the cell-centered viscosity on all levels + * filling the specified container of Old or New LevelData + * \param a_time either Old or New stamp + */ void calcViscosity(const PeleLM::TimeStamp &a_time); + + /** + * \brief Compute the face-centered turbulent viscosity on all levels + * filling the specified container of Old or New LevelData + * \param a_time either Old or New stamp + */ void calcTurbViscosity(const PeleLM::TimeStamp &a_time); + + /** + * \brief Compute the cell-centereds diffusivity on all levels + * filling the specified container of Old or New LevelData + * \param a_time either Old or New stamp + */ void calcDiffusivity(const PeleLM::TimeStamp &a_time); // get edge-centered diffusivity on a per level / per comp basis + /** + * \brief Compute face-averaged diffusivity (or else) from + * cell-centered data, accounting for Dirichlet BCs, allowing + * to call user-defined zero_visc() function and possibly adding + * a turbulent contribution + * \param lev target level + * \param beta_comp first component to use in a_diff_cc MultiFab + * \param ncomp number of components of the outgoing data + * \param doZeroVisc zero_visc() call trigger + * \param a_bcrec boundary conditions for the required components + * \param a_diff_cc cell-centered data + * \param addTurbContrib turbulent contribution increment trigger + */ amrex::Array getDiffusivity(int lev, int beta_comp, int ncomp, int doZeroVisc, amrex::Vector a_bcrec, const amrex::MultiFab &a_diff_cc, int addTurbContrib = 0); - // get species/enthalpy differential diffusion fluxes + /** + * \brief Compute the explicit face-centered diffusion fluxes on all levels + * for species and enthalpy (NUM_SPECIES+2 components) using Old or + * New state/coefficient data from the LevelData container + * \param a_time either Old or New stamp + * \param a_fluxes outgoing fluxes container, the first NUM_SPECIES components filled + * \param a_EBfluxes optional EB fluxes container + * \param a_wbarfluxes optional Wbar fluxes container + * \param a_soretfluxes optional Soret fluxes container + */ void computeDifferentialDiffusionFluxes(const PeleLM::TimeStamp &a_time, const amrex::Vector> &a_fluxes, const amrex::Vector &a_EBfluxes, const amrex::Vector> &a_wbarfluxes, const amrex::Vector> &a_soretfluxes); - // get species/enthalpy differential diffusion terms + /** + * \brief Compute the explicit cell-centered diffusion term on all levels + * by computing the divergence of the diffusion fluxes at Old or New time, + * filling the proper container in diffData + * \param a_time either Old or New stamp + * \param diffData container for the returning diffusion terms + * \param is_init initial computation flag + */ void computeDifferentialDiffusionTerms(const PeleLM::TimeStamp &a_time, std::unique_ptr &diffData, int is_init = 0); - // add wbar term to the species fluxes + /** + * \brief Add the Wbar contribution to the face-centered species diffusion fluxes on all levels + * optionnally returning that contribution in a separate container + * \param a_fluxes diffusion fluxes to be updated + * \param a_wbarfluxes optional container to return the Wbar contribution + * \param a_spec species rhoYs state data on all levels + * \param a_rho density state data on all levels + * \param a_beta cell-centered species diffusivity on all levels + */ void addWbarTerm(const amrex::Vector> &a_fluxes, const amrex::Vector> &a_wbarfluxes, amrex::Vector const &a_spec, amrex::Vector const &a_rho, amrex::Vector const &a_beta); - // add soret term to the species flux + /** + * \brief Add the Soret contribution to the face-centered species diffusion fluxes on all levels + * optionnally returning that contribution in a separate container + * \param a_fluxes diffusion fluxes to be updated + * \param a_soretfluxes optional container to return the Soret contribution + * \param a_spec species rhoYs state data on all levels + * \param a_rho density state data on all levels + * \param a_temp temperature state data on all levels + * \param a_beta cell-centered species diffusivity on all levels + */ void addSoretTerm(const amrex::Vector> &a_fluxes, const amrex::Vector> &a_soretfluxes, amrex::Vector const &a_spec, @@ -367,23 +564,56 @@ class PeleLM : public amrex::AmrCore { amrex::Vector const &a_temp, amrex::Vector const &a_beta); - // Ensure sum of specues diffusion fluxes sum to 0 + /** + * \brief Modify the species diffusion fluxes by computing a correction velocity, + * ensuring they sum up to zero. + * \param a_fluxes diffusion fluxes to be updated + * \param a_spec species rhoYs state data on all levels + */ void adjustSpeciesFluxes(const amrex::Vector> &a_fluxes, amrex::Vector const &a_spec); - // Compute enthalpy flux due to species diffusion + /** + * \brief Compute the enthalpy flux due to species differential diffusion, + * filling the NUM_SPECIES+1 component of a_fluxes + * \param a_fluxes outgoing fluxes container, filled on the NUM_SPECIES+1 component + * \param a_temp temperature state data on all levels + */ void computeSpeciesEnthalpyFlux(const amrex::Vector> &a_fluxes, amrex::Vector const &a_temp); + /** + * \brief Implicit diffusion solves + * \param advData container for the external forcing of the linear solve + * \param diffData container for the returning implicit diffusion term + */ void differentialDiffusionUpdate(std::unique_ptr &advData, std::unique_ptr &diffData); + /** + * \brief Setup the implicit deltaT linear solve + * \param a_rhs outgoing RHS for the linear solve + * \param a_Tsave outgoing temporary container for temperature while solving for deltaT + * \param a_rhoCp outgoing linear solve coefficient + * \param advData container for external forcing used to build RHS + * \param diffData container for diffusion forcing used to build RHS + */ void deltaTIter_prepare(const amrex::Vector &a_rhs, const amrex::Vector &a_Tsave, const amrex::Vector &a_rhoCp, std::unique_ptr &advData, std::unique_ptr &diffData); + /** + * \brief Restore state, update enthalpy and compute fluxes after deltaT linear solve + * within the deltaT iteration scheme + * \param a_dtiter deltaT iteration index + * \param a_fluxes diffusion fluxes, with updated Fourier and differential species fluxes + * \param a_ebfluxes optional temperature fluxes at the EB + * \param a_Tsave temporary container to restore state temperature + * \param diffData container for diffusion forcing used to build RHS + * \param a_deltaT_norm norm of deltaT residual + */ void deltaTIter_update(int a_dtiter, const amrex::Vector> &a_fluxes, const amrex::Vector &a_ebfluxes, @@ -392,20 +622,39 @@ class PeleLM : public amrex::AmrCore { amrex::Real &a_deltaT_norm); - // Viscous forces + /** + * \brief Compute the viscous force term in the momentum equation using AMReX + * TensorOp + * \param a_time state time for velocity, density and viscosity, either Old or New + * \param a_divtau outgoing divTau term + * \param use_density flag to scale divTau by 1/rho + * \param scale prefactor of divTau, used to weight by num. scheme property (e.g. 0.5 for + * Crank Nicholson) + */ void computeDivTau(const PeleLM::TimeStamp &a_time, const amrex::Vector &a_divtau, int use_density, amrex::Real scale = 1.0); - // Diffuse velocity + /** + * \brief Implicit diffusion of the of velocity field using AMReX + * TensorOp. Always act on New time data and include Crank Nicholson factor + */ void diffuseVelocity(); - // diffusion Linear operator BC + /** + * \brief Return LinOpBC for the scalar diffusion operator + * \param a_side AMReX orientaion + * \param a_bc BCRec for the scalar of interest + */ amrex::Array getDiffusionLinOpBC(amrex::Orientation::Side a_side, const amrex::BCRec &a_bc); - // diffusion Tensor operator BC + /** + * \brief Return AMReX_SPACEDIM LinOpBC vector for the velocity tensor operator + * \param a_side AMReX orientaion + * \param a_bc BCRec for all the velocity components + */ amrex::Vector> getDiffusionTensorOpBC(amrex::Orientation::Side a_side, const amrex::Vector a_bc); @@ -414,30 +663,84 @@ class PeleLM : public amrex::AmrCore { //----------------------------------------------------------------------------- // REACTION + /** + * \brief Top-level reaction solve function, performing the chemistry integration + * on all the AMR levels. + * \param advData container for Forcing, the advection/diffusion component of the + * advance function in the SDC context + */ void advanceChemistry(std::unique_ptr &advData); + /** + * \brief Performing the chemistry integration on a given level, for a fixed + * step length and relying on AmrCore BoxArray/DMap, not masking fine-covered regions + * \param lev level of interest + * \param a_dt integration length + * \param a_extForcing advection/diffusion forcing + */ void advanceChemistry(int lev, const amrex::Real &a_dt, amrex::MultiFab &a_extForcing); + /** + * \brief Performing the chemistry integration on a given level, for a fixed + * step length and using the BoxArray/DMap masking fine-covered region and + * specifically load balanced for reaction + * \param lev level of interest + * \param a_dt integration length + * \param a_extForcing advection/diffusion forcing + */ void advanceChemistryBAChem(int lev, const amrex::Real &a_dt, amrex::MultiFab &a_extForcing); + /** + * \brief Top-level instantaneous reaction rate function, acting on all levels + * \param a_I_R outgoing multi-level container inst. RR container + * \param a_time state time used to compute RR, either Old or New + */ void computeInstantaneousReactionRate(const amrex::Vector &a_I_R, const PeleLM::TimeStamp &a_time); + /** + * \brief Compute instantaneous reaction rate on a given level + * \param lev level of interest + * \param a_time state time used to compute RR, either Old or New + * \param a_I_R outgoing MultiFab for inst. RR container + */ void computeInstantaneousReactionRate(int lev, const PeleLM::TimeStamp &a_time, amrex::MultiFab* a_I_R); + + /** + * \brief Compute the heat release rate on a given level, using the + * LevelDataReact I_R data + * \param lev level of interest + * \param a_HR outgoing MultiFab + */ void getHeatRelease(int lev, amrex::MultiFab *a_HR); //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- // Time step size + /** + * \brief Compute the time step sive accross all levels and processes. + * \param is_init flag to trigger init-only dt factors + * \param a_time state time to work on, either Old or New + */ amrex::Real computeDt(int is_init, const PeleLM::TimeStamp &a_time); + + /** + * \brief Compute the CFL step size + * \param a_time state time to work on, either Old or New + */ amrex::Real estConvectiveDt(const PeleLM::TimeStamp &a_time); + + /** + * \brief Compute the divU step size + * \param a_time state time to work on, either Old or New + */ amrex::Real estDivUDt(const PeleLM::TimeStamp &a_time); //----------------------------------------------------------------------------- @@ -519,34 +822,83 @@ class PeleLM : public amrex::AmrCore { //----------------------------------------------------------------------------- #endif + //----------------------------------------------------------------------------- // EOS - // Thermodynamic pressure + /** + * \brief Compute thermodynamic pressure (RhoRT) from state on all levels + * \param a_time state time to work on, either Old or New + */ void setThermoPress(const PeleLM::TimeStamp &a_time); + + /** + * \brief Compute thermodynamic pressure (RhoRT) from state on a single level + * \param lev level of interest + * \param a_time state time to work on, either Old or New + */ void setThermoPress(int lev, const PeleLM::TimeStamp &a_time); - // Enforce rho/rhoYs consistency + /** + * \brief Enforce rho = \sum rhoYs on a given level + * \param lev level of interest + * \param a_time state time to work on, either Old or New + */ void setRhoToSumRhoY(int lev, const PeleLM::TimeStamp &a_time); - // update Temperature from rhoY and rhoH + /** + * \brief Compute temperature from rhoY and rhoH on all levels + * \param a_time state time to work on, either Old or New + */ void setTemperature(const PeleLM::TimeStamp &a_time); + + /** + * \brief Compute temperature from rhoY and rhoH on a given level + * \param lev level of interest + * \param a_time state time to work on, either Old or New + */ void setTemperature(int lev, const PeleLM::TimeStamp &a_time); - // Divergence constraint + /** + * \brief Compute the divergence constraint from reaction/diffusion + * terms + * \param is_init flag for init-only checks + * \param computeDiff flag to trigger computing the diffusion terms or + * using the one already precompute and available in diffData + * \param a_time state time to work with + * \param diffData container for the diffusion terms + */ void calcDivU(int is_init, int computeDiff, int do_avgDown, const PeleLM::TimeStamp &a_time, std::unique_ptr &diffData); - // Pressure drift + /** + * \brief Compute the pressure drift term using RhoRT and + * state data at a_time, acting on all levels + * \param a_time state time to work on, either Old or New + * \param a_dPdt outgoing container + */ void calc_dPdt(const PeleLM::TimeStamp &a_time, const amrex::Vector &a_dPdt); + + /** + * \brief Compute the pressure drift term using RhoRT and + * state data at a_time on a given level + * \param lev level of interest + * \param a_time state time to work on, either Old or New + * \param a_dPdt outgoing container + */ void calc_dPdt(int lev, const PeleLM::TimeStamp &a_time, amrex::MultiFab* a_dPdt); - // Closed Chamber + /** + * \brief For the closed chamber algorithm, compute averaged + * mac_divU and theta, update divU and the spatially averaged + * thermodynamic pressure. Return the mean divU. + * \param advData container including mac_divu + */ amrex::Real adjustPandDivU(std::unique_ptr &advData); //----------------------------------------------------------------------------- @@ -563,7 +915,6 @@ class PeleLM : public amrex::AmrCore { void WriteHeader(const std::string &name, bool is_checkpoint) const; void WriteDebugPlotFile(const amrex::Vector &a_MF, const std::string &pltname); - void initLevelDataFromPlt(int a_lev, const std::string &a_pltFile); //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- @@ -662,6 +1013,7 @@ class PeleLM : public amrex::AmrCore { //----------------------------------------------------------------------------- // UTILS + template void parseUserKey(amrex::ParmParse &a_pp, const std::string &a_search_key, @@ -781,9 +1133,6 @@ class PeleLM : public amrex::AmrCore { void floorSpecies(const PeleLM::TimeStamp &a_time); - void loadBalanceChem(); - void loadBalanceChemLev(int a_lev); - amrex::Real MLNorm0(const amrex::Vector &a_MF); amrex::Vector MLNorm0(const amrex::Vector &a_MF, @@ -825,6 +1174,9 @@ class PeleLM : public amrex::AmrCore { int &nComp, const std::string &a_var); + void evaluateChemExtForces(const amrex::Vector &a_chemForces); + void evaluateAdvectionTerms(const amrex::Vector &a_advTerms); + // Lightweight descriptors access bool isStateVariable(std::string_view a_name); bool isReactVariable(std::string_view a_name); @@ -1123,7 +1475,7 @@ class PeleLM : public amrex::AmrCore { // External source terms amrex::Vector > m_extSource; - // Covered cells mask + // Fine-covered cells mask amrex::Vector > m_coveredMask; int m_resetCoveredMask; diff --git a/Source/PeleLMEvaluate.cpp b/Source/PeleLMEvaluate.cpp index 30f102b8..a24b90e9 100644 --- a/Source/PeleLMEvaluate.cpp +++ b/Source/PeleLMEvaluate.cpp @@ -18,7 +18,6 @@ void PeleLM::Evaluate() { if ( !itexists ) { amrex::Error("PeleLM::evaluate(): unknown variable: "+m_evaluatePlotVars[ivar]); } - Print() << " --> Evaluating " << m_evaluatePlotVars[ivar] << "\n"; if ( derive_lst.canDerive(m_evaluatePlotVars[ivar]) ) { const PeleLMDeriveRec* rec = derive_lst.get(m_evaluatePlotVars[ivar]); ncomp += rec->numDerive(); @@ -44,12 +43,20 @@ void PeleLM::Evaluate() { mf_plt[lev].define(grids[lev], dmap[lev], ncomp, 0, MFInfo(), Factory(lev)); } + //---------------------------------------------------------------- + // Prepare a few things if not restarting from a chkfile + if (m_restart_chkfile.empty()) { + m_nstep = 0; + } + //---------------------------------------------------------------- // Fill the outgoing container int cnt = 0; for (int ivar = 0; ivar < m_evaluatePlotVarCount; ivar++ ) { int cntIncr = 0; + Print() << " --> Evaluating " << m_evaluatePlotVars[ivar] << "\n"; + // Evaluate function calls actual PeleLM::Evolve pieces and may require // the entire multi-level hierarchy if ( evaluate_lst.canDerive(m_evaluatePlotVars[ivar]) ) { @@ -73,6 +80,9 @@ void PeleLM::Evaluate() { // Write the evaluated variables to disc Vector istep(finest_level + 1, 0); + // Override m_cur_time to store the dt in pltEvaluate + m_cur_time = m_dt; + std::string plotfilename = "pltEvaluate"; amrex::WriteMultiLevelPlotfile(plotfilename, finest_level + 1, GetVecOfConstPtrs(mf_plt), plt_VarsName, Geom(), m_cur_time, istep, refRatio()); @@ -150,8 +160,30 @@ PeleLM::MLevaluate(const Vector &a_MFVec, } nComp = NUM_SPECIES+2; } else if ( a_var == "advTerm" ) { - // TODO - Abort("advTerm not available yet, soon hopefully"); + Vector > aliasMF(finest_level+1); + for (int lev = 0; lev <= finest_level; ++lev) { + aliasMF[lev] = std::make_unique (*a_MFVec[lev], amrex::make_alias, a_comp, NVAR-2); + } + evaluateAdvectionTerms(GetVecOfPtrs(aliasMF)); + nComp = NVAR-2; + } else if ( a_var == "chemTest" ) { + // 'Old' chemical state (rhoYs + rhoH + T) and adv/diff forcing for chem. integration + // Replicate most of the advance function + // Copy the state + for (int lev = 0; lev <= finest_level; ++lev) { + auto ldata_p = getLevelDataPtr(lev,AmrNewTime); + MultiFab::Copy(*a_MFVec[lev],ldata_p->state,FIRSTSPEC,a_comp,NUM_SPECIES+2,0); + } + // Inital velocity projection + if (m_restart_chkfile.empty()) { + projectInitSolution(); + } + Vector > aliasMF(finest_level+1); + for (int lev = 0; lev <= finest_level; ++lev) { + aliasMF[lev] = std::make_unique (*a_MFVec[lev], amrex::make_alias, a_comp+NUM_SPECIES+2, NUM_SPECIES+1); + } + evaluateChemExtForces(GetVecOfPtrs(aliasMF)); + nComp = 2*(NUM_SPECIES+1)+1; } else if ( a_var == "instRR" ) { for (int lev = 0; lev <= finest_level; ++lev) { std::unique_ptr I_RR = std::make_unique (*a_MFVec[lev],amrex::make_alias,a_comp,NUM_SPECIES); @@ -187,3 +219,194 @@ PeleLM::MLevaluate(const Vector &a_MFVec, nComp = AMREX_SPACEDIM; } } + +void +PeleLM::evaluateChemExtForces(const amrex::Vector &a_chemForces) +{ + //---------------------------------------------------------------- + // Copy old <- new state + copyStateNewToOld(1); + copyPressNewToOld(); + + //---------------------------------------------------------------- + // TIME + // Compute time-step size + m_dt = computeDt(0,AmrOldTime); + + // Update time vectors + for(int lev = 0; lev <= finest_level; lev++) + { + m_t_old[lev] = m_cur_time; + m_t_new[lev] = m_cur_time + m_dt; + } + //---------------------------------------------------------------- + + //---------------------------------------------------------------- + // Data for the advance, only live for the duration of the advance + std::unique_ptr diffData; + diffData.reset(new AdvanceDiffData(finest_level, grids, dmap, m_factory, m_nGrowAdv, m_use_wbar, m_use_soret)); + std::unique_ptr advData; + advData.reset(new AdvanceAdvData(finest_level, grids, dmap, m_factory, m_incompressible, + m_nGrowAdv, m_nGrowMAC)); + + //---------------------------------------------------------------- + // Advance setup + // Pre-SDC + m_sdcIter = 0; + + // fillpatch the t^{n} data + averageDownState(AmrOldTime); + fillPatchState(AmrOldTime); + + // compute t^{n} data + calcViscosity(AmrOldTime); + calcDiffusivity(AmrOldTime); + + floorSpecies(AmrOldTime); + setThermoPress(AmrOldTime); + computeDifferentialDiffusionTerms(AmrOldTime,diffData); + + // Init SDC iteration + copyTransportOldToNew(); + copyDiffusionOldToNew(diffData); + + // Compute the instantaneous reaction rate + // on Old state + computeInstantaneousReactionRate(getIRVect(),AmrOldTime); + + //---------------------------------------------------------------- + // Perform a single SDC iteration + m_sdcIter = 1; + + // Predict face velocity with Godunov + predictVelocity(advData); + + // Create S^{n+1/2} by fillpatching t^{n} and t^{np1,k} + createMACRHS(advData); + + // Re-evaluate thermo. pressure and add chi_increment + addChiIncrement(m_sdcIter, AmrNewTime, advData); + + // MAC projection + macProject(AmrOldTime,advData,GetVecOfPtrs(advData->mac_divu)); + + // Get scalar advection SDC forcing + getScalarAdvForce(advData,diffData); + + // Get AofS: (\nabla \cdot (\rho Y Umac))^{n+1/2,k} + // and for density = \sum_k AofS_k + computeScalarAdvTerms(advData); + + // Compute \rho^{np1,k+1} and fillpatch new density + updateDensity(advData); + fillPatchDensity(AmrNewTime); + + // Get scalar diffusion SDC RHS (stored in Forcing) + getScalarDiffForce(advData,diffData); + + // Diffuse scalars + differentialDiffusionUpdate(advData,diffData); + + // Get external forcing for chemistry + getScalarReactForce(advData); + + // Copy external forcing for chemistry into outgoing container + for (int lev = 0; lev <= finest_level; ++lev) { + MultiFab::Copy(*a_chemForces[lev],advData->Forcing[lev],0,0,NUM_SPECIES+1,0); + } + + // Reset state + copyStateOldToNew(); +} + +void +PeleLM::evaluateAdvectionTerms(const amrex::Vector &a_advTerms) +{ + //---------------------------------------------------------------- + // Copy old <- new state + copyStateNewToOld(1); + copyPressNewToOld(); + + //---------------------------------------------------------------- + // TIME + // Compute time-step size + m_dt = computeDt(0,AmrOldTime); + + // Update time vectors + for(int lev = 0; lev <= finest_level; lev++) + { + m_t_old[lev] = m_cur_time; + m_t_new[lev] = m_cur_time + m_dt; + } + //---------------------------------------------------------------- + + //---------------------------------------------------------------- + // Data for the advance, only live for the duration of the advance + std::unique_ptr diffData; + diffData.reset(new AdvanceDiffData(finest_level, grids, dmap, m_factory, m_nGrowAdv, m_use_wbar, m_use_soret)); + std::unique_ptr advData; + advData.reset(new AdvanceAdvData(finest_level, grids, dmap, m_factory, m_incompressible, + m_nGrowAdv, m_nGrowMAC)); + + //---------------------------------------------------------------- + // Advance setup + // Pre-SDC + m_sdcIter = 0; + + // fillpatch the t^{n} data + averageDownState(AmrOldTime); + fillPatchState(AmrOldTime); + + // compute t^{n} data + calcViscosity(AmrOldTime); + calcDiffusivity(AmrOldTime); + + floorSpecies(AmrOldTime); + setThermoPress(AmrOldTime); + computeDifferentialDiffusionTerms(AmrOldTime,diffData); + + // Init SDC iteration + copyTransportOldToNew(); + copyDiffusionOldToNew(diffData); + + // Compute the instantaneous reaction rate + // on Old state + computeInstantaneousReactionRate(getIRVect(),AmrOldTime); + + //---------------------------------------------------------------- + // Perform a single SDC iteration + m_sdcIter = 1; + + // Predict face velocity with Godunov + predictVelocity(advData); + + // Create S^{n+1/2} by fillpatching t^{n} and t^{np1,k} + createMACRHS(advData); + WriteDebugPlotFile(GetVecOfConstPtrs(advData->mac_divu),"macdivU"); + + // Re-evaluate thermo. pressure and add chi_increment + addChiIncrement(m_sdcIter, AmrNewTime, advData); + WriteDebugPlotFile(GetVecOfConstPtrs(advData->mac_divu),"macdivUwithIncr"); + + // MAC projection + macProject(AmrOldTime,advData,GetVecOfPtrs(advData->mac_divu)); + + // Get scalar advection SDC forcing + getScalarAdvForce(advData,diffData); + WriteDebugPlotFile(GetVecOfConstPtrs(advData->Forcing),"advForcing"); + + // Get AofS: (\nabla \cdot (\rho Y Umac))^{n+1/2,k} + // and for density = \sum_k AofS_k + computeScalarAdvTerms(advData); + + // Compute t^{n+1/2} velocity advection term + computeVelocityAdvTerm(advData); + + // Copy AofS into outgoing container, skip Temperature and RhoRT + for (int lev = 0; lev <= finest_level; ++lev) { + MultiFab::Copy(*a_advTerms[lev],advData->AofS[lev],0,0,NVAR-2,0); + } + + // Reset state + copyStateOldToNew(); +} diff --git a/Source/PeleLMInit.cpp b/Source/PeleLMInit.cpp index 2b65af60..e806ba6a 100644 --- a/Source/PeleLMInit.cpp +++ b/Source/PeleLMInit.cpp @@ -150,7 +150,6 @@ void PeleLM::initData() { BL_PROFILE_VAR("PeleLM::initData()", initData); if (m_restart_chkfile.empty()) { - //---------------------------------------------------------------- if (!m_initial_grid_file.empty()) { InitFromGridFile(m_cur_time); @@ -192,111 +191,16 @@ void PeleLM::initData() { return; } -#ifdef PELE_USE_EFIELD - poissonSolveEF(AmrNewTime); - fillPatchPhiV(AmrNewTime); -#endif - - // Post data Init time step estimate - Real dtInit = computeDt(is_init,AmrNewTime); - Print() << " Initial dt: " << dtInit << "\n"; - - if (m_do_init_proj) { - - Print() << "\n Doing initial projection(s) \n\n"; - // Subcycling IAMR/PeleLM first does a projection with no reaction divU - // which can make the dt for evaluating I_R better - if (m_has_divu) { - int is_initialization = 1; // Yes we are - int computeDiffusionTerm = 1; // Needed here - int do_avgDown = 1; // Always - - // Light version of the diffusion data container - std::unique_ptr diffData; - diffData.reset(new AdvanceDiffData(finest_level, grids, dmap, m_factory, - m_nGrowAdv, m_use_wbar, m_use_soret, is_initialization)); - calcDivU(is_initialization,computeDiffusionTerm,do_avgDown,AmrNewTime,diffData); - } - initialProjection(); - - // If gravity is used, do initial pressure projection to get the hydrostatic pressure - if (std::abs(m_gravity.sum()) > 0.0) { - initialPressProjection(); - } - - // Post data init time step estimate - m_dt = computeDt(is_init,AmrNewTime); - Print() << " Initial dt: " << m_dt << "\n"; - - //---------------------------------------------------------------- - // Initial velocity projection iterations - for (int iter = 0; iter < m_numDivuIter; iter++) { - if (m_do_react) { - // The new level data has been filled above - // Copy new -> old since old used in advanceChemistry - copyStateNewToOld(); - - for (int lev = finest_level; lev >= 0; --lev) { - // Setup empty forcing - MultiFab Forcing(grids[lev],dmap[lev],nCompForcing(),0); - Forcing.setVal(0.0); - - if (lev != finest_level) { - advanceChemistryBAChem(lev, dtInit/2.0, Forcing); - } else { - if (m_max_grid_size_chem.min() > 0) { - advanceChemistryBAChem(lev, dtInit/2.0, Forcing); - } else { - advanceChemistry(lev, dtInit/2.0, Forcing); - } - } - if (m_doLoadBalance) { - loadBalanceChemLev(lev); - } - } - // Copy back old -> new - copyStateOldToNew(); - } - if (m_has_divu) { - int is_initialization = 1; // Yes we are - int computeDiffusionTerm = 1; // Needed here - int do_avgDown = 1; // Always - - // Light version of the diffusion data container - std::unique_ptr diffData; - diffData.reset(new AdvanceDiffData(finest_level, grids, dmap, m_factory, - m_nGrowAdv, m_use_wbar, m_use_soret, is_initialization)); - calcDivU(is_initialization,computeDiffusionTerm,do_avgDown,AmrNewTime,diffData); - } - initialProjection(); - } - - if ( m_numDivuIter == 0 && m_do_react ) { - for (int lev = 0; lev <= finest_level; ++lev) { - auto ldataR_p = getLevelDataReactPtr(lev); - ldataR_p->I_R.setVal(0.0); - } - } - Print() << PrettyLine; - } else { - // If we didn't do the projection, initialize press/gp(/I_R) - for (int lev = 0; lev <= finest_level; ++lev) { - if ( m_do_react ) { - auto ldataR_p = getLevelDataReactPtr(lev); - ldataR_p->I_R.setVal(0.0); - } - auto ldata_p = getLevelDataPtr(lev,AmrNewTime); - ldata_p->press.setVal(0.0); - ldata_p->gp.setVal(0.0); - } - } + //---------------------------------------------------------------- + // Project initial solution + projectInitSolution(); // Active control int is_restart = 0; activeControl(is_restart); //---------------------------------------------------------------- - // Initial pressure iterations + // Do initial pressure iterations initialIterations(); m_nstep = 0; @@ -425,6 +329,110 @@ void PeleLM::initLevelData(int lev) { } } +void PeleLM::projectInitSolution() +{ + const int is_init = 1; + +#ifdef PELE_USE_EFIELD + poissonSolveEF(AmrNewTime); + fillPatchPhiV(AmrNewTime); +#endif + + // Post data Init time step estimate + Real dtInit = computeDt(is_init,AmrNewTime); + Print() << " Initial dt: " << dtInit << "\n"; + + if (m_do_init_proj) { + + Print() << "\n Doing initial projection(s) \n\n"; + // Subcycling IAMR/PeleLM first does a projection with no reaction divU + // which can make the dt for evaluating I_R better + if (m_has_divu) { + int is_initialization = 1; // Yes we are + int computeDiffusionTerm = 1; // Needed here + int do_avgDown = 1; // Always + + // Light version of the diffusion data container + std::unique_ptr diffData; + diffData.reset(new AdvanceDiffData(finest_level, grids, dmap, m_factory, + m_nGrowAdv, m_use_wbar, m_use_soret, is_initialization)); + calcDivU(is_initialization,computeDiffusionTerm,do_avgDown,AmrNewTime,diffData); + } + initialProjection(); + + // If gravity is used, do initial pressure projection to get the hydrostatic pressure + if (std::abs(m_gravity.sum()) > 0.0) { + initialPressProjection(); + } + + // Post data init time step estimate + m_dt = computeDt(is_init,AmrNewTime); + Print() << " Initial dt: " << m_dt << "\n"; + + //---------------------------------------------------------------- + // Initial velocity projection iterations + for (int iter = 0; iter < m_numDivuIter; iter++) { + if (m_do_react) { + // The new level data has been filled above + // Copy new -> old since old used in advanceChemistry + copyStateNewToOld(); + + for (int lev = finest_level; lev >= 0; --lev) { + // Setup empty forcing + MultiFab Forcing(grids[lev],dmap[lev],nCompForcing(),0); + Forcing.setVal(0.0); + + if (lev != finest_level) { + advanceChemistryBAChem(lev, dtInit/2.0, Forcing); + } else { + if (m_max_grid_size_chem.min() > 0) { + advanceChemistryBAChem(lev, dtInit/2.0, Forcing); + } else { + advanceChemistry(lev, dtInit/2.0, Forcing); + } + } + if (m_doLoadBalance) { + loadBalanceChemLev(lev); + } + } + // Copy back old -> new + copyStateOldToNew(); + } + if (m_has_divu) { + int is_initialization = 1; // Yes we are + int computeDiffusionTerm = 1; // Needed here + int do_avgDown = 1; // Always + + // Light version of the diffusion data container + std::unique_ptr diffData; + diffData.reset(new AdvanceDiffData(finest_level, grids, dmap, m_factory, + m_nGrowAdv, m_use_wbar, m_use_soret, is_initialization)); + calcDivU(is_initialization,computeDiffusionTerm,do_avgDown,AmrNewTime,diffData); + } + initialProjection(); + } + + if ( m_numDivuIter == 0 && m_do_react ) { + for (int lev = 0; lev <= finest_level; ++lev) { + auto ldataR_p = getLevelDataReactPtr(lev); + ldataR_p->I_R.setVal(0.0); + } + } + Print() << PrettyLine; + } else { + // If we didn't do the projection, initialize press/gp(/I_R) + for (int lev = 0; lev <= finest_level; ++lev) { + if ( m_do_react ) { + auto ldataR_p = getLevelDataReactPtr(lev); + ldataR_p->I_R.setVal(0.0); + } + auto ldata_p = getLevelDataPtr(lev,AmrNewTime); + ldata_p->press.setVal(0.0); + ldata_p->gp.setVal(0.0); + } + } +} + void PeleLM::initialIterations() { BL_PROFILE_VAR("PeleLM::initialIterations()", initialIterations); diff --git a/Source/PeleLMSetup.cpp b/Source/PeleLMSetup.cpp index 1f2d236c..da855a01 100644 --- a/Source/PeleLMSetup.cpp +++ b/Source/PeleLMSetup.cpp @@ -1010,15 +1010,33 @@ void PeleLM::evaluateSetup() evaluate_lst.add("diffTerm",IndexType::TheCellType(),NUM_SPECIES+2,var_names,the_same_box); } - // scalar advection term + // advection terms { - // TODO - Vector var_names(NUM_SPECIES+1); + Vector var_names(NVAR-2); // Skip temperature and RhoRT, unused + AMREX_D_TERM(var_names[VELX] = "A(VELX)";, + var_names[VELY] = "A(VELY)";, + var_names[VELZ] = "A(VELZ)"); + var_names[DENSITY] = "A(Rho)"; for (int n = 0 ; n < NUM_SPECIES; n++) { - var_names[n] = "A("+spec_names[n]+")"; + var_names[FIRSTSPEC+n] = "A("+spec_names[n]+")"; } - var_names[NUM_SPECIES] = "A(RhoH)"; - evaluate_lst.add("advTerm",IndexType::TheCellType(),NUM_SPECIES+1,var_names,the_same_box); + var_names[RHOH] = "A(RhoH)"; + evaluate_lst.add("advTerm",IndexType::TheCellType(),NVAR-2,var_names,the_same_box); + } + + // Chemical state and external chem. forcing (used in ReactEval) + { + Vector var_names(2*(NUM_SPECIES+1)+1); + for (int n = 0 ; n < NUM_SPECIES; n++) { + var_names[n] = "rhoY("+spec_names[n]+")"; + } + var_names[NUM_SPECIES] = "rhoH"; + var_names[NUM_SPECIES+1] = "Temp"; + for (int n = 0 ; n < NUM_SPECIES; n++) { + var_names[NUM_SPECIES+2+n] = "F_rhoY("+spec_names[n]+")"; + } + var_names[2*NUM_SPECIES+2] = "F_rhoH"; + evaluate_lst.add("chemTest",IndexType::TheCellType(),2*(NUM_SPECIES+1)+1,var_names,the_same_box); } // instantaneous reaction rate