Skip to content

Commit

Permalink
some fixes for 2D
Browse files Browse the repository at this point in the history
  • Loading branch information
Hariswaran Sitaraman committed Sep 27, 2024
1 parent 17d4525 commit 4ad1095
Show file tree
Hide file tree
Showing 10 changed files with 144 additions and 119 deletions.
2 changes: 1 addition & 1 deletion Docs/sphinx/geometry/EB.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ Rotational frames
------------------------------

.. warning:: PeleC has limited support for simulations using rotating frames using the single-reference-frame (SRF)
method and needs the MOL scheme when using this feature.
method only in 3D and needs the MOL scheme when using this feature.


Single-reference-frame implementation
Expand Down
16 changes: 13 additions & 3 deletions Source/Derive.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,20 +92,30 @@ PeleC::pc_dervel_if(
{
auto const dat = datfab.const_array();
auto vel = derfab.array();
int rotframeflag=do_rf;

int axis = rf_axis;
amrex::Real omega = rf_omega;
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {
rf_axis_x, rf_axis_y, rf_axis_z};
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {AMREX_D_DECL(
rf_axis_x, rf_axis_y, rf_axis_z)};
auto const gdata = geomdata.data();

amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
if(rotframeflag) {
#if AMREX_SPACEDIM > 2
amrex::RealVect r = get_rotaxis_vec(i, j, k, axis_loc, gdata);
amrex::RealVect w(0.0, 0.0, 0.0);
amrex::RealVect w(AMREX_D_DECL(0.0, 0.0, 0.0));
w[axis] = omega;
amrex::RealVect w_cross_r = w.crossProduct(r);
vel(i, j, k) =
dat(i, j, k, UMX + dir) / dat(i, j, k, URHO) + w_cross_r[dir];
#endif
}
else {
vel(i, j, k) =
dat(i, j, k, UMX + dir) / dat(i, j, k, URHO);
}

});
}

Expand Down
164 changes: 88 additions & 76 deletions Source/Derive.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,102 +46,114 @@ PeleC::pc_dermagvel_if(
auto magvel = derfab.array();

int axis = rf_axis;
int rotframeflag=do_rf;
amrex::Real omega = rf_omega;
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {
rf_axis_x, rf_axis_y, rf_axis_z};
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {AMREX_D_DECL(
rf_axis_x, rf_axis_y, rf_axis_z)};
auto const gdata = geomdata.data();

amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
amrex::RealVect r = get_rotaxis_vec(i, j, k, axis_loc, gdata);
amrex::RealVect w(0.0, 0.0, 0.0);
w[axis] = omega;
amrex::RealVect w_cross_r = w.crossProduct(r);

const amrex::Real datinv = 1.0 / dat(i, j, k, URHO);
const amrex::Real dat1 = dat(i, j, k, UMX) * datinv + w_cross_r[0];
const amrex::Real dat2 = dat(i, j, k, UMY) * datinv + w_cross_r[1];
const amrex::Real dat3 = dat(i, j, k, UMZ) * datinv + w_cross_r[2];
magvel(i, j, k) = sqrt((dat1 * dat1) + (dat2 * dat2) + (dat3 * dat3));
if(rotframeflag) {
#if AMREX_SPACEDIM > 2
amrex::RealVect r = get_rotaxis_vec(i, j, k, axis_loc, gdata);
amrex::RealVect w(0.0, 0.0, 0.0);
w[axis] = omega;
amrex::RealVect w_cross_r = w.crossProduct(r);

const amrex::Real datinv = 1.0 / dat(i, j, k, URHO);
const amrex::Real dat1 = dat(i, j, k, UMX) * datinv + w_cross_r[0];
const amrex::Real dat2 = dat(i, j, k, UMY) * datinv + w_cross_r[1];
const amrex::Real dat3 = dat(i, j, k, UMZ) * datinv + w_cross_r[2];
magvel(i, j, k) = sqrt((dat1 * dat1) + (dat2 * dat2) + (dat3 * dat3));
#endif
}
else {
const amrex::Real datinv = 1.0 / dat(i, j, k, URHO);
const amrex::Real dat1 = dat(i, j, k, UMX) * datinv;
const amrex::Real dat2 = dat(i, j, k, UMY) * datinv;
const amrex::Real dat3 = dat(i, j, k, UMZ) * datinv;
magvel(i, j, k) = sqrt((dat1 * dat1) + (dat2 * dat2) + (dat3 * dat3));
}
});
}

void
pc_dermagmom(
const amrex::Box& bx,
amrex::FArrayBox& derfab,
int /*dcomp*/,
int /*ncomp*/,
const amrex::FArrayBox& datfab,
const amrex::Geometry& /*geomdata*/,
amrex::Real /*time*/,
const int* /*bcrec*/,
const int /*level*/)
const amrex::Box& bx,
amrex::FArrayBox& derfab,
int /*dcomp*/,
int /*ncomp*/,
const amrex::FArrayBox& datfab,
const amrex::Geometry& /*geomdata*/,
amrex::Real /*time*/,
const int* /*bcrec*/,
const int /*level*/)
{
auto const dat = datfab.const_array();
auto magmom = derfab.array();

amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
magmom(i, j, k) = sqrt(
dat(i, j, k, UMX) * dat(i, j, k, UMX) +
dat(i, j, k, UMY) * dat(i, j, k, UMY) +
dat(i, j, k, UMZ) * dat(i, j, k, UMZ));
});
auto const dat = datfab.const_array();
auto magmom = derfab.array();

amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
magmom(i, j, k) = sqrt(
dat(i, j, k, UMX) * dat(i, j, k, UMX) +
dat(i, j, k, UMY) * dat(i, j, k, UMY) +
dat(i, j, k, UMZ) * dat(i, j, k, UMZ));
});
}

void
pc_derkineng(
const amrex::Box& bx,
amrex::FArrayBox& derfab,
int /*dcomp*/,
int /*ncomp*/,
const amrex::FArrayBox& datfab,
const amrex::Geometry& /*geomdata*/,
amrex::Real /*time*/,
const int* /*bcrec*/,
const int /*level*/)
const amrex::Box& bx,
amrex::FArrayBox& derfab,
int /*dcomp*/,
int /*ncomp*/,
const amrex::FArrayBox& datfab,
const amrex::Geometry& /*geomdata*/,
amrex::Real /*time*/,
const int* /*bcrec*/,
const int /*level*/)
{
auto const dat = datfab.const_array();
auto kineng = derfab.array();

amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
const amrex::Real datxsq = dat(i, j, k, UMX) * dat(i, j, k, UMX);
const amrex::Real datysq = dat(i, j, k, UMY) * dat(i, j, k, UMY);
const amrex::Real datzsq = dat(i, j, k, UMZ) * dat(i, j, k, UMZ);
kineng(i, j, k) = 0.5 / dat(i, j, k, URHO) * (datxsq + datysq + datzsq);
});
auto const dat = datfab.const_array();
auto kineng = derfab.array();

amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
const amrex::Real datxsq = dat(i, j, k, UMX) * dat(i, j, k, UMX);
const amrex::Real datysq = dat(i, j, k, UMY) * dat(i, j, k, UMY);
const amrex::Real datzsq = dat(i, j, k, UMZ) * dat(i, j, k, UMZ);
kineng(i, j, k) = 0.5 / dat(i, j, k, URHO) * (datxsq + datysq + datzsq);
});
}

void
pc_dereint1(
const amrex::Box& bx,
amrex::FArrayBox& derfab,
int /*dcomp*/,
int /*ncomp*/,
const amrex::FArrayBox& datfab,
const amrex::Geometry& /*geomdata*/,
amrex::Real /*time*/,
const int* /*bcrec*/,
const int /*level*/)
// amrex::Real omega=0.0,
// amrex::Real rad=0.0 )
const amrex::Box& bx,
amrex::FArrayBox& derfab,
int /*dcomp*/,
int /*ncomp*/,
const amrex::FArrayBox& datfab,
const amrex::Geometry& /*geomdata*/,
amrex::Real /*time*/,
const int* /*bcrec*/,
const int /*level*/)
// amrex::Real omega=0.0,
// amrex::Real rad=0.0 )
{
// FIXME:Hari S:
// can't seem to add omega and rad
// as derive functions are quite rigid
// Compute internal energy from (rho E).
auto const dat = datfab.const_array();
auto e = derfab.array();

amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
const amrex::Real rhoInv = 1.0 / dat(i, j, k, URHO);
const amrex::Real ux = dat(i, j, k, UMX) * rhoInv;
const amrex::Real uy = dat(i, j, k, UMY) * rhoInv;
const amrex::Real uz = dat(i, j, k, UMZ) * rhoInv;
e(i, j, k) =
dat(i, j, k, UEDEN) * rhoInv - 0.5 * (ux * ux + uy * uy + uz * uz);
// see Blazek, Appendix A3, Navier-Stokes in rotating frame of reference
// e(i,j,k)-=0.5*omega*omega*rad*rad;
});
// FIXME:Hari S:
// can't seem to add omega and rad
// as derive functions are quite rigid
// Compute internal energy from (rho E).
auto const dat = datfab.const_array();
auto e = derfab.array();

amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
const amrex::Real rhoInv = 1.0 / dat(i, j, k, URHO);
const amrex::Real ux = dat(i, j, k, UMX) * rhoInv;
const amrex::Real uy = dat(i, j, k, UMY) * rhoInv;
const amrex::Real uz = dat(i, j, k, UMZ) * rhoInv;
e(i, j, k) =
dat(i, j, k, UEDEN) * rhoInv - 0.5 * (ux * ux + uy * uy + uz * uz);
// see Blazek, Appendix A3, Navier-Stokes in rotating frame of reference
// e(i,j,k)-=0.5*omega*omega*rad*rad;
});
}

void
Expand Down
4 changes: 2 additions & 2 deletions Source/Diffusion.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ PeleC::getMOLSrcTerm(
int using_rf = do_rf;
amrex::Real omega = rf_omega;
int axis = rf_axis;
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {
rf_axis_x, rf_axis_y, rf_axis_z};
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {AMREX_D_DECL(
rf_axis_x, rf_axis_y, rf_axis_z)};
const auto dx = geom.CellSizeArray();

/*
Expand Down
12 changes: 7 additions & 5 deletions Source/InitEB.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ PeleC::initialize_eb2_structs()
sv_eb_flux.resize(vfrac.local_size());
sv_eb_bcval.resize(vfrac.local_size());

amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {
rf_axis_x, rf_axis_y, rf_axis_z};
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {AMREX_D_DECL(
rf_axis_x, rf_axis_y, rf_axis_z)};
amrex::Real rfdist2 = rf_rad * rf_rad;

auto const& flags = ebfactory.getMultiEBCellFlagFab();
Expand Down Expand Up @@ -183,21 +183,22 @@ PeleC::initialize_eb2_structs()
amrex::ParallelFor(ncutcells, [=] AMREX_GPU_DEVICE(int L) {
const auto& iv = sten[L].iv;

#if AMREX_SPACEDIM > 2
amrex::Real xloc =
prob_lo[0] + (iv[0] + 0.5 + sten[L].eb_centroid[0]) * dxlev[0];
amrex::Real yloc =
prob_lo[1] + (iv[1] + 0.5 + sten[L].eb_centroid[1]) * dxlev[1];
amrex::Real zloc =
prob_lo[2] + (iv[2] + 0.5 + sten[L].eb_centroid[2]) * dxlev[2];

amrex::RealVect r(0.0, 0.0, 0.0);
amrex::RealVect w(0.0, 0.0, 0.0);
amrex::RealVect r(AMRE_D_DECL(0.0, 0.0, 0.0));
amrex::RealVect w(AMREX_D_DECL(0.0, 0.0, 0.0));

r[0] = xloc - axis_loc[0];
r[1] = yloc - axis_loc[1];
r[2] = zloc - axis_loc[2];

amrex::Real rmag2 = r[0] * r[0] + r[1] * r[1] + r[2] * r[2];
amrex::Real rmag2 = r.radSquared();
rmag2 -= r[axis] * r[axis]; // only in plane

if (rmag2 > rfdist2) {
Expand All @@ -212,6 +213,7 @@ PeleC::initialize_eb2_structs()
v_bcval[L] = 0.0;
w_bcval[L] = 0.0;
}
#endif
});
}
} else {
Expand Down
5 changes: 2 additions & 3 deletions Source/MOL.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ pc_compute_hyp_mol_flux(
const auto geomdata = geom.data();
int axisdir_captured = axisdir;
amrex::Real omega_captured = omega;
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axisloc_captured = {
axis_loc[0], axis_loc[1], axis_loc[2]};
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axisloc_captured = {AMREX_D_DECL(
axis_loc[0], axis_loc[1], axis_loc[2])};

for (int dir = 0; dir < AMREX_SPACEDIM; dir++) {
amrex::FArrayBox dq_fab(cbox, QVAR, amrex::The_Async_Arena());
Expand Down Expand Up @@ -136,7 +136,6 @@ pc_compute_hyp_mol_flux(
amrex::Real flux_tmp[NVAR] = {0.0};
amrex::Real ustar = 0.0;

amrex::RealVect r(0.0, 0.0, 0.0);
amrex::Real radl, radr;
if (using_rotframe) {

Expand Down
8 changes: 4 additions & 4 deletions Source/PeleC.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1547,8 +1547,8 @@ PeleC::enforce_consistent_e(amrex::MultiFab& S)
const auto geomdata = geom.data();
int axis = rf_axis;
amrex::Real omega = rf_omega;
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {
rf_axis_x, rf_axis_y, rf_axis_z};
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {AMREX_D_DECL(
rf_axis_x, rf_axis_y, rf_axis_z)};

amrex::ParallelFor(
S, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept {
Expand Down Expand Up @@ -2208,8 +2208,8 @@ PeleC::reset_internal_energy(amrex::MultiFab& S_new, int ng)
const auto geomdata = geom.data();
int axis = rf_axis;
amrex::Real omega = rf_omega;
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {
rf_axis_x, rf_axis_y, rf_axis_z};
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {AMREX_D_DECL(
rf_axis_x, rf_axis_y, rf_axis_z)};
amrex::ParallelFor(
S_new, ngs, [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k) noexcept {
amrex::Real rad = get_rotaxis_dist(i, j, k, axis, axis_loc, geomdata);
Expand Down
6 changes: 3 additions & 3 deletions Source/React.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,12 +123,12 @@ PeleC::react_state(
auto const& flags = fact.getMultiEBCellFlagFab();

// for rotational frames
amrex::Real rotframeflag = do_rf;
int rotframeflag = do_rf;
const auto geomdata = geom.data();
int axis = rf_axis;
amrex::Real omega = rf_omega;
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {
rf_axis_x, rf_axis_y, rf_axis_z};
amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> axis_loc = {AMREX_D_DECL(
rf_axis_x, rf_axis_y, rf_axis_z)};

#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
Expand Down
9 changes: 6 additions & 3 deletions Source/RotSource.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ PeleC::fill_rot_source(
amrex::MultiFab& rot_src,
int ng)
{
#if AMREX_SPACEDIM > 2
auto const& fact =
dynamic_cast<amrex::EBFArrayBoxFactory const&>(state_old.Factory());
auto const& flags = fact.getMultiEBCellFlagFab();
Expand All @@ -66,9 +67,9 @@ PeleC::fill_rot_source(
const auto& sarr = sarrs[nbx];
const auto& src = srcs[nbx];

amrex::RealVect r(0.0, 0.0, 0.0);
amrex::RealVect w(0.0, 0.0, 0.0);
amrex::RealVect v(0.0, 0.0, 0.0);
amrex::RealVect r(AMREX_D_DECL(0.0, 0.0, 0.0));
amrex::RealVect w(AMREX_D_DECL(0.0, 0.0, 0.0));
amrex::RealVect v(AMREX_D_DECL(0.0, 0.0, 0.0));

r = get_rotaxis_vec(i, j, k, axis_loc, geomdata);
w[axis] = omega;
Expand All @@ -87,11 +88,13 @@ PeleC::fill_rot_source(
-sarr(i, j, k, URHO) * (2.0 * w_cross_v[0] + w_cross_w_cross_r[0]);
src(i, j, k, UMY) =
-sarr(i, j, k, URHO) * (2.0 * w_cross_v[1] + w_cross_w_cross_r[1]);

src(i, j, k, UMZ) =
-sarr(i, j, k, URHO) * (2.0 * w_cross_v[2] + w_cross_w_cross_r[2]);
}
}
});

amrex::Gpu::synchronize();
#endif
}
Loading

0 comments on commit 4ad1095

Please sign in to comment.