parthenon-hpc-lab · Yurlungur · Sep 26, 2020 · Sep 17, 2020 · Sep 18, 2020 · Sep 18, 2020
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,7 @@
 
 ### Changed (changing behavior/API/variables/...)
 - [[PR 303]](https://github.com/lanl/parthenon/pull/303) Changed `Mesh::BlockList` from a `std::list<MeshBlock>` to a `std::vector<std::shared_ptr<MeshBlock>>`, making `FindMeshBlock` run in constant, rather than linear, time. Loops over `block_list` in application drivers must be cahnged accordingly.
+- [[PR 307]](https://github.com/lanl/parthenon/pull/307) Changed back-pointers in mesh structure to weak pointers. Cleaned up `MeshBlock` constructor and implemented `MeshBlock` factory function.
 
 ### Fixed (not changing behavior/API/variables/...)
 - [[PR 293]](https://github.com/lanl/parthenon/pull/293) Changed `VariablePack` and related objects to use `ParArray1D` objects instead of `ParArrayND` objects under the hood to reduce the size of the captured objects.

diff --git a/example/advection/advection_driver.cpp b/example/advection/advection_driver.cpp
@@ -124,7 +124,7 @@ TaskList AdvectionDriver::MakeTaskList(MeshBlock *pmb, int stage) {
   if (stage == integrator->nstages) {
     auto new_dt = tl.AddTask(
         [](std::shared_ptr<Container<Real>> &rc) {
-          MeshBlock *pmb = rc->pmy_block;
+          auto pmb = rc->GetBlockPointer();
           pmb->SetBlockTimestep(parthenon::Update::EstimateTimestep(rc));
           return TaskStatus::complete;
         },

diff --git a/example/advection/advection_package.cpp b/example/advection/advection_package.cpp
@@ -14,6 +14,7 @@
 #include <algorithm>
 #include <cmath>
 #include <limits>
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -163,7 +164,7 @@ std::shared_ptr<StateDescriptor> Initialize(ParameterInput *pin) {
 }
 
 AmrTag CheckRefinement(std::shared_ptr<Container<Real>> &rc) {
-  MeshBlock *pmb = rc->pmy_block;
+  auto pmb = rc->GetBlockPointer();
   // refine on advected, for example.  could also be a derived quantity
   auto v = rc->Get("advected").data;
 
@@ -195,7 +196,7 @@ AmrTag CheckRefinement(std::shared_ptr<Container<Real>> &rc) {
 
 // demonstrate usage of a "pre" fill derived routine
 void PreFill(std::shared_ptr<Container<Real>> &rc) {
-  MeshBlock *pmb = rc->pmy_block;
+  auto pmb = rc->GetBlockPointer();
 
   IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire);
   IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire);
@@ -215,7 +216,7 @@ void PreFill(std::shared_ptr<Container<Real>> &rc) {
 
 // this is the package registered function to fill derived
 void SquareIt(std::shared_ptr<Container<Real>> &rc) {
-  MeshBlock *pmb = rc->pmy_block;
+  auto pmb = rc->GetBlockPointer();
 
   IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire);
   IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire);
@@ -235,7 +236,7 @@ void SquareIt(std::shared_ptr<Container<Real>> &rc) {
 
 // demonstrate usage of a "post" fill derived routine
 void PostFill(std::shared_ptr<Container<Real>> &rc) {
-  MeshBlock *pmb = rc->pmy_block;
+  auto pmb = rc->GetBlockPointer();
 
   IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire);
   IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire);
@@ -258,7 +259,7 @@ void PostFill(std::shared_ptr<Container<Real>> &rc) {
 
 // provide the routine that estimates a stable timestep for this package
 Real EstimateTimestep(std::shared_ptr<Container<Real>> &rc) {
-  MeshBlock *pmb = rc->pmy_block;
+  auto pmb = rc->GetBlockPointer();
   auto pkg = pmb->packages["advection_package"];
   const auto &cfl = pkg->Param<Real>("cfl");
   const auto &vx = pkg->Param<Real>("vx");
@@ -293,7 +294,7 @@ Real EstimateTimestep(std::shared_ptr<Container<Real>> &rc) {
 // some field "advected" that we are pushing around.
 // This routine implements all the "physics" in this example
 TaskStatus CalculateFluxes(std::shared_ptr<Container<Real>> &rc) {
-  MeshBlock *pmb = rc->pmy_block;
+  auto pmb = rc->GetBlockPointer();
   IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior);
   IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior);
   IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior);

diff --git a/example/calculate_pi/calculate_pi.cpp b/example/calculate_pi/calculate_pi.cpp
@@ -36,7 +36,7 @@ using namespace parthenon::package::prelude;
 namespace calculate_pi {
 
 void SetInOrOut(std::shared_ptr<Container<Real>> &rc) {
-  MeshBlock *pmb = rc->pmy_block;
+  auto pmb = rc->GetBlockPointer();
   IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior);
   IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior);
   IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior);

diff --git a/example/kokkos_pi/kokkos_pi.cpp b/example/kokkos_pi/kokkos_pi.cpp
@@ -179,7 +179,7 @@ static BlockList_t setupMesh(const int &n_block, const int &n_mesh, const double
         h_xyz(2, idx) = dxyzCell * (static_cast<Real>(k_mesh * n_block) + 0.5) - delta;
         // Add variable for in_or_out
         auto &base = pmb->real_containers.Get();
-        base->setBlock(pmb.get());
+        base->SetBlockPointer(pmb);
         base->Add("in_or_out", myMetadata);
       }
     }

diff --git a/src/bvals/boundary_conditions.cpp b/src/bvals/boundary_conditions.cpp
@@ -23,7 +23,7 @@
 namespace parthenon {
 
 TaskStatus ApplyBoundaryConditions(std::shared_ptr<Container<Real>> &rc) {
-  MeshBlock *pmb = rc->pmy_block;
+  std::shared_ptr<MeshBlock> pmb = rc->GetBlockPointer();
   const IndexDomain interior = IndexDomain::interior;
   const IndexDomain entire = IndexDomain::entire;
   IndexRange ib = pmb->cellbounds.GetBoundsI(interior);

diff --git a/src/bvals/bvals.cpp b/src/bvals/bvals.cpp
@@ -46,9 +46,11 @@ namespace parthenon {
 // BoundaryValues constructor (the first object constructed inside the MeshBlock()
 // constructor): sets functions for the appropriate boundary conditions at each of the 6
 // dirs of a MeshBlock
-BoundaryValues::BoundaryValues(MeshBlock *pmb, BoundaryFlag *input_bcs,
+BoundaryValues::BoundaryValues(std::weak_ptr<MeshBlock> wpmb, BoundaryFlag *input_bcs,
                                ParameterInput *pin)
-    : BoundaryBase(pmb->pmy_mesh, pmb->loc, pmb->block_size, input_bcs), pmy_block_(pmb) {
+    : BoundaryBase(wpmb.lock()->pmy_mesh, wpmb.lock()->loc, wpmb.lock()->block_size,
+                   input_bcs),
+      pmy_block_(wpmb) {
   // Check BC functions for each of the 6 boundaries in turn ---------------------
   for (int i = 0; i < 6; i++) {
     switch (block_bcs[i]) {
@@ -66,6 +68,7 @@ BoundaryValues::BoundaryValues(MeshBlock *pmb, BoundaryFlag *input_bcs,
   CheckBoundaryFlag(block_bcs[BoundaryFace::inner_x1], CoordinateDirection::X1DIR);
   CheckBoundaryFlag(block_bcs[BoundaryFace::outer_x1], CoordinateDirection::X1DIR);
 
+  std::shared_ptr<MeshBlock> pmb = GetBlockPointer();
   if (pmb->block_size.nx2 > 1) {
     nface_ = 4;
     nedge_ = 4;

diff --git a/src/bvals/bvals.hpp b/src/bvals/bvals.hpp
@@ -28,6 +28,7 @@
 #include "bvals/bvals_interfaces.hpp"
 #include "defs.hpp"
 #include "parthenon_arrays.hpp"
+#include "utils/error_checking.hpp"
 
 namespace parthenon {
 
@@ -98,7 +99,8 @@ class BoundaryBase {
 class BoundaryValues : public BoundaryBase, // public BoundaryPhysics,
                        public BoundaryCommunication {
  public:
-  BoundaryValues(MeshBlock *pmb, BoundaryFlag *input_bcs, ParameterInput *pin);
+  BoundaryValues(std::weak_ptr<MeshBlock> pmb, BoundaryFlag *input_bcs,
+                 ParameterInput *pin);
 
   // variable-length arrays of references to BoundaryVariable instances
   // containing all BoundaryVariable instances:
@@ -128,8 +130,9 @@ class BoundaryValues : public BoundaryBase, // public BoundaryPhysics,
   int AdvanceCounterPhysID(int num_phys);
 
  private:
-  MeshBlock *pmy_block_; // ptr to MeshBlock containing this BoundaryValues
-  int nface_, nedge_;    // used only in fc/flux_correction_fc.cpp calculations
+  // ptr to MeshBlock containing this BoundaryValues
+  std::weak_ptr<MeshBlock> pmy_block_;
+  int nface_, nedge_; // used only in fc/flux_correction_fc.cpp calculations
 
   // if a BoundaryPhysics or user fn should be applied at each MeshBlock boundary
   // false --> e.g. block, polar, periodic boundaries
@@ -149,6 +152,14 @@ class BoundaryValues : public BoundaryBase, // public BoundaryPhysics,
   void ProlongateGhostCells(const NeighborBlock &nb, int si, int ei, int sj, int ej,
                             int sk, int ek);
 
+  /// Returns shared pointer to a block
+  std::shared_ptr<MeshBlock> GetBlockPointer() {
+    if (pmy_block_.expired()) {
+      PARTHENON_THROW("Invalid pointer to MeshBlock!");
+    }
+    return pmy_block_.lock();
+  }
+
   // temporary--- Added by @tomidakn on 2015-11-27 in f0f989f85f
   // TODO(KGF): consider removing this friendship designation
   friend class Mesh;

diff --git a/src/bvals/bvals_interfaces.hpp b/src/bvals/bvals_interfaces.hpp
@@ -22,13 +22,15 @@
 // TODO(felker): deduplicate forward declarations
 // TODO(felker): consider moving enums and structs in a new file? bvals_structs.hpp?
 
+#include <memory>
 #include <string>
 #include <vector>
 
 #include "parthenon_mpi.hpp"
 
 #include "defs.hpp"
 #include "parthenon_arrays.hpp"
+#include "utils/error_checking.hpp"
 
 namespace parthenon {
 
@@ -250,7 +252,7 @@ class BoundaryBuffer {
 
 class BoundaryVariable : public BoundaryCommunication, public BoundaryBuffer {
  public:
-  explicit BoundaryVariable(MeshBlock *pmb);
+  explicit BoundaryVariable(std::weak_ptr<MeshBlock> pmb);
   virtual ~BoundaryVariable() = default;
 
   // (usuallly the std::size_t unsigned integer type)
@@ -270,9 +272,18 @@ class BoundaryVariable : public BoundaryCommunication, public BoundaryBuffer {
   BoundaryData<> bd_var_, bd_var_flcor_;
   // derived class dtors are also responsible for calling DestroyBoundaryData(bd_var_)
 
-  MeshBlock *pmy_block_; // ptr to MeshBlock containing this BoundaryVariable
+  // ptr to MeshBlock containing this BoundaryVariable
+  std::weak_ptr<MeshBlock> pmy_block_;
   Mesh *pmy_mesh_;
 
+  /// Returns shared pointer to a block
+  std::shared_ptr<MeshBlock> GetBlockPointer() {
+    if (pmy_block_.expired()) {
+      PARTHENON_THROW("Invalid pointer to MeshBlock!");
+    }
+    return pmy_block_.lock();
+  }
+
   void CopyVariableBufferSameProcess(NeighborBlock &nb, int ssize);
   void CopyFluxCorrectionBufferSameProcess(NeighborBlock &nb, int ssize);
 

diff --git a/src/bvals/bvals_refine.cpp b/src/bvals/bvals_refine.cpp
@@ -75,7 +75,7 @@ namespace parthenon {
 // (automatically switches back to conserved variables at the end of fn)
 
 void BoundaryValues::ProlongateBoundaries(const Real time, const Real dt) {
-  MeshBlock *pmb = pmy_block_;
+  std::shared_ptr<MeshBlock> pmb = GetBlockPointer();
   int &mylevel = pmb->loc.level;
 
   // This hardcoded technique is also used to manually specify the coupling between
@@ -202,7 +202,7 @@ void BoundaryValues::ProlongateBoundaries(const Real time, const Real dt) {
 
 void BoundaryValues::RestrictGhostCellsOnSameLevel(const NeighborBlock &nb, int nk,
                                                    int nj, int ni) {
-  MeshBlock *pmb = pmy_block_;
+  std::shared_ptr<MeshBlock> pmb = GetBlockPointer();
   MeshRefinement *pmr = pmb->pmr.get();
 
   const IndexDomain interior = IndexDomain::interior;
@@ -294,7 +294,7 @@ void BoundaryValues::ApplyPhysicalBoundariesOnCoarseLevel(const NeighborBlock &n
 
 void BoundaryValues::ProlongateGhostCells(const NeighborBlock &nb, int si, int ei, int sj,
                                           int ej, int sk, int ek) {
-  MeshBlock *pmb = pmy_block_;
+  std::shared_ptr<MeshBlock> pmb = GetBlockPointer();
   auto &pmr = pmb->pmr;
 
   for (auto cc_pair : pmr->pvars_cc_) {

diff --git a/src/bvals/bvals_var.cpp b/src/bvals/bvals_var.cpp
@@ -33,15 +33,15 @@
 
 namespace parthenon {
 
-BoundaryVariable::BoundaryVariable(MeshBlock *pmb)
-    : bvar_index(), pmy_block_(pmb), pmy_mesh_(pmb->pmy_mesh) {}
+BoundaryVariable::BoundaryVariable(std::weak_ptr<MeshBlock> pmb)
+    : bvar_index(), pmy_block_(pmb), pmy_mesh_(pmb.lock()->pmy_mesh) {}
 
 //----------------------------------------------------------------------------------------
 //! \fn void BoundaryVariable::InitBoundaryData(BoundaryData<> &bd, BoundaryQuantity type)
 //  \brief Initialize BoundaryData structure
 
 void BoundaryVariable::InitBoundaryData(BoundaryData<> &bd, BoundaryQuantity type) {
-  MeshBlock *pmb = pmy_block_;
+  auto pmb = GetBlockPointer();
   NeighborIndexes *ni = pmb->pbval->ni;
   int cng = pmb->cnghost;
   int size = 0;
@@ -138,7 +138,7 @@ void BoundaryVariable::CopyFluxCorrectionBufferSameProcess(NeighborBlock &nb, in
 //  \brief Send boundary buffers of variables
 
 void BoundaryVariable::SendBoundaryBuffers() {
-  MeshBlock *pmb = pmy_block_;
+  auto pmb = GetBlockPointer();
   int mylevel = pmb->loc.level;
   for (int n = 0; n < pmb->pbval->nneighbor; n++) {
     NeighborBlock &nb = pmb->pbval->neighbor[n];
@@ -173,8 +173,9 @@ void BoundaryVariable::SendBoundaryBuffers() {
 bool BoundaryVariable::ReceiveBoundaryBuffers() {
   bool bflag = true;
 
-  for (int n = 0; n < pmy_block_->pbval->nneighbor; n++) {
-    NeighborBlock &nb = pmy_block_->pbval->neighbor[n];
+  auto pmb = GetBlockPointer();
+  for (int n = 0; n < pmb->pbval->nneighbor; n++) {
+    NeighborBlock &nb = pmb->pbval->neighbor[n];
     if (bd_var_.flag[nb.bufid] == BoundaryStatus::arrived) continue;
     if (bd_var_.flag[nb.bufid] == BoundaryStatus::waiting) {
       if (nb.snb.rank == Globals::my_rank) { // on the same process
@@ -203,7 +204,7 @@ bool BoundaryVariable::ReceiveBoundaryBuffers() {
 //  \brief set the boundary data
 
 void BoundaryVariable::SetBoundaries() {
-  MeshBlock *pmb = pmy_block_;
+  auto pmb = GetBlockPointer();
   int mylevel = pmb->loc.level;
   for (int n = 0; n < pmb->pbval->nneighbor; n++) {
     NeighborBlock &nb = pmb->pbval->neighbor[n];
@@ -225,7 +226,7 @@ void BoundaryVariable::SetBoundaries() {
 //  \brief receive and set the boundary data for initialization
 
 void BoundaryVariable::ReceiveAndSetBoundariesWithWait() {
-  MeshBlock *pmb = pmy_block_;
+  auto pmb = GetBlockPointer();
   int mylevel = pmb->loc.level;
   pmb->exec_space.fence();
   for (int n = 0; n < pmb->pbval->nneighbor; n++) {