diff --git a/src/basis.cpp b/src/basis.cpp index a6b3aab..eb7a17f 100644 --- a/src/basis.cpp +++ b/src/basis.cpp @@ -126,7 +126,7 @@ Mat_unique build_basis_matrix( PetscErrorCode perr = MatCreateMPIAIJWithArrays( comm, idxn.size() - 1, PETSC_DECIDE, m_size, n_size, idxn.data(), idxm.data(), mdat.data(), m_basis.get()); - CHKERRABORT(comm, perr); + CHKERRXX(perr); #ifdef DEBUG_VERBOSE matrix_dbg_print(comm, *m_basis, "Basis Matrix"); @@ -174,7 +174,7 @@ Mat_unique build_warp_matrix( // lambda needed here anyway to capture comm auto get_raw_array = [comm](floating*& a, const Vec* v) -> void { PetscErrorCode p = VecGetArray(*v, &a); - CHKERRABORT(comm, p); + CHKERRXX(p); }; n_ary_for_each(get_raw_array, raw_arrs.begin(), raw_arrs.end(), displacements.begin()); @@ -237,7 +237,7 @@ Mat_unique build_warp_matrix( // lambda needed here anyway to capture comm auto restore_raw_array = [comm](floating*& a, const Vec* v) -> void { PetscErrorCode p = VecRestoreArray(*v, &a); - CHKERRABORT(comm, p); + CHKERRXX(p); }; n_ary_for_each(restore_raw_array, raw_arrs.begin(), raw_arrs.end(), displacements.begin()); @@ -245,7 +245,7 @@ Mat_unique build_warp_matrix( perr = MatCreateMPIAIJWithArrays( comm, idxn.size() - 1, idxn.size() - 1, mat_size, mat_size, idxn.data(), idxm.data(), mdat.data(), warp.get()); - CHKERRABORT(comm, perr); + CHKERRXX(perr); debug_creation(*warp, "Warp matrix"); return warp; diff --git a/src/elastic.cpp b/src/elastic.cpp index 3986195..b8f3311 100644 --- a/src/elastic.cpp +++ b/src/elastic.cpp @@ -138,9 +138,9 @@ void Elastic::innerloop(integer outer_count) // check convergence and break if below threshold floating posmax, negmax; PetscErrorCode perr = VecMax(*m_workspace->m_delta, nullptr, &posmax); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecMin(*m_workspace->m_delta, nullptr, &negmax); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); floating amax = std::max(std::fabs(posmax), std::fabs(negmax)); PetscPrintf(m_comm, "Maximum displacement: %.2f\n", amax); floating aavg; @@ -169,7 +169,7 @@ void Elastic::innerstep(integer inum, bool recalculate_lambda) // TODO: can we reuse here? PetscErrorCode perr = MatTransposeMatMult(*m_workspace->m_tmat, *m_workspace->m_tmat, MAT_INITIAL_MATRIX, PETSC_DEFAULT, normmat.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); debug_creation(*normmat, std::string("Mat_normal") + std::to_string(inum)); // precondition tmat2 block_precondition(*normmat, m_p_map->size(), m_p_map->m_ndim); @@ -189,15 +189,15 @@ void Elastic::innerstep(integer inum, bool recalculate_lambda) floating total_mult = lapl_mult * lambda_mult * m_lambda; // calculate tmat2 + lambda*lapl2 perr = MatAXPY(*normmat, total_mult, *m_p_map->laplacian(), DIFFERENT_NONZERO_PATTERN); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // calculate rvec, to do this need to reuse stacked vector for [f-m f-m f-m f-m] perr = VecWAXPY( *m_workspace->m_globaltmps[0], -1.0, *m_p_registered->global_vec(), *m_fixed.global_vec()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); m_workspace->duplicate_single_grad_to_stacked(0); perr = MatMultTranspose(*m_workspace->m_tmat, *m_workspace->m_stacktmp, *m_workspace->m_rhs); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // apply memory term if needed if (configuration.grab("with_memory")) @@ -205,15 +205,15 @@ void Elastic::innerstep(integer inum, bool recalculate_lambda) // build -lambda*a Vec_unique disp = create_unique_vec(); perr = VecDuplicate(m_p_map->displacements(), disp.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecCopy(m_p_map->displacements(), *disp); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecScale(*disp, -total_mult); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // calculate vecdot(lapl_2, -lambda*a) and add to rhs in one operation perr = MatMultAdd(*m_p_map->laplacian(), *disp, *m_workspace->m_rhs, *m_workspace->m_rhs); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } // Force free tmat as no longer needed @@ -222,17 +222,17 @@ void Elastic::innerstep(integer inum, bool recalculate_lambda) // solve for delta a KSP_unique m_ksp = create_unique_ksp(); perr = KSPCreate(m_comm, m_ksp.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = KSPSetOperators(*m_ksp, *normmat, *normmat); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = KSPSetUp(*m_ksp); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = KSPSetFromOptions(*m_ksp); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = KSPSetUp(*m_ksp); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = KSPSolve(*m_ksp, *m_workspace->m_rhs, *m_workspace->m_delta); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // update map m_p_map->update(*m_workspace->m_delta); // warp image @@ -260,19 +260,19 @@ void Elastic::calculate_tmat(integer iternum __attribute__((unused))) // Calculate average intensity 0.5(f+m) // Constant offset needed later, does not affect gradients PetscErrorCode perr = VecSet(*m_workspace->m_globaltmps[m_fixed.ndim()], -1.0); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); // NB Z = aX + bY + cZ has call signature VecAXPBYPCZ(Z, a, b, c, X, Y) because reasons.... perr = VecAXPBYPCZ(*m_workspace->m_globaltmps[m_fixed.ndim()], 0.5, 0.5, 1, *m_fixed.global_vec(), *m_p_registered->global_vec()); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); // scatter this to local for later perr = DMGlobalToLocalBegin(*m_fixed.dmda(), *m_workspace->m_globaltmps[m_fixed.ndim()], INSERT_VALUES, *m_workspace->m_localtmp); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = DMGlobalToLocalEnd(*m_fixed.dmda(), *m_workspace->m_globaltmps[m_fixed.ndim()], INSERT_VALUES, *m_workspace->m_localtmp); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // find average gradients for (uinteger idim = 0; idim < m_fixed.ndim(); idim++) @@ -284,7 +284,7 @@ void Elastic::calculate_tmat(integer iternum __attribute__((unused))) // Negate average intensity to get 1 - 0.5(f+m) as needed by algorithm perr = VecScale(*m_workspace->m_globaltmps[m_fixed.ndim()], -1.0); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); // scatter grads into stacked vector m_workspace->scatter_grads_to_stacked(); @@ -292,12 +292,12 @@ void Elastic::calculate_tmat(integer iternum __attribute__((unused))) // 3. copy basis into p_tmat m_workspace->m_tmat = create_unique_mat(); perr = MatDuplicate(*m_p_map->basis(), MAT_COPY_VALUES, m_workspace->m_tmat.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); debug_creation(*m_workspace->m_tmat, std::string("Mat_tmat_") + std::to_string(iternum)); // 4. left diagonal multiply p_tmat with stacked vector perr = MatDiagonalScale(*m_workspace->m_tmat, *m_workspace->m_stacktmp, nullptr); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } void Elastic::save_debug_map(integer outer_count, integer inner_count) @@ -438,19 +438,19 @@ floating Elastic::approximate_optimum_lambda(Mat& mat_a, Mat& mat_b, floating la if (recalc_first) { perr = MatAXPY(*mat_c, lambda_mult * x_lo, mat_b, DIFFERENT_NONZERO_PATTERN); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); y_lo = get_condnum_by_poweriter(*mat_c, 0.01, 100); } recalc_first = true; // Calculate at initial guess perr = MatAXPY(*mat_c, lambda_mult * (x_mid - x_lo), mat_b, DIFFERENT_NONZERO_PATTERN); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); y_mid = get_condnum_by_poweriter(*mat_c, 0.01, 100); // calculate at higher value perr = MatAXPY(*mat_c, lambda_mult * (x_hi - x_mid), mat_b, DIFFERENT_NONZERO_PATTERN); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); y_hi = get_condnum_by_poweriter(*mat_c, 0.01, 100); PetscPrintf( diff --git a/src/fd_routines.cpp b/src/fd_routines.cpp index 2c54937..c67f8df 100644 --- a/src/fd_routines.cpp +++ b/src/fd_routines.cpp @@ -23,32 +23,32 @@ Vec_unique fd::gradient_to_global_unique(const DM &dmda, const Vec &localvec, in // (should have matching global, local and comm size) Vec dm_local_vec; PetscErrorCode perr = DMGetLocalVector(dmda, &dm_local_vec); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); if (!vecs_equivalent(dm_local_vec, localvec)) { throw InternalError("provided vector invalid for given dmda object", __FILE__, __LINE__); } perr = DMRestoreLocalVector(dmda, &dm_local_vec); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); // New global vec must be a dmda global Vec_unique grad = create_unique_vec(); perr = DMCreateGlobalVector(dmda, grad.get()); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); // Access local of this to have ghosts, global of grad to avoid later copy // Recall grad and image share a DMDA floating ***img_array, ***grad_array; // acceptable to use raw ptrs here as memory handled by PETSc perr = DMDAVecGetArray(dmda, localvec, &img_array); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); perr = DMDAVecGetArray(dmda, *(grad.get()), &grad_array); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); integer i_lo, i_hi, j_lo, j_hi, k_lo, k_hi; // This returns corners + widths so add lo to get hi perr = DMDAGetCorners(dmda, &i_lo, &j_lo, &k_lo, &i_hi, &j_hi, &k_hi); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); i_hi += i_lo; j_hi += j_lo; k_hi += k_lo; @@ -70,9 +70,9 @@ Vec_unique fd::gradient_to_global_unique(const DM &dmda, const Vec &localvec, in } // Release pointers and allow petsc to cleanup perr = DMDAVecRestoreArray(dmda, localvec, &img_array); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); perr = DMDAVecRestoreArray(dmda, *(grad.get()), &grad_array); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); return grad; } @@ -83,27 +83,27 @@ void fd::gradient_existing(const DM &dmda, const Vec &srcvec, Vec &tgtvec, integ // (should have matching global, local and comm size) Vec dm_local_vec; PetscErrorCode perr = DMGetLocalVector(dmda, &dm_local_vec); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); if (!vecs_equivalent(dm_local_vec, srcvec)) { throw InternalError("provided srcvec invalid for given dmda object", __FILE__, __LINE__); } perr = DMRestoreLocalVector(dmda, &dm_local_vec); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); // Access local of this to have ghosts, global of grad to avoid later copy // Recall grad and image share a DMDA floating ***img_array, ***grad_array; // acceptable to use raw ptrs here as memory handled by PETSc perr = DMDAVecGetArray(dmda, srcvec, &img_array); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); perr = DMDAVecGetArray(dmda, tgtvec, &grad_array); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); integer i_lo, i_hi, j_lo, j_hi, k_lo, k_hi; // This returns corners + widths so add lo to get hi perr = DMDAGetCorners(dmda, &i_lo, &j_lo, &k_lo, &i_hi, &j_hi, &k_hi); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); i_hi += i_lo; j_hi += j_lo; k_hi += k_lo; @@ -125,7 +125,7 @@ void fd::gradient_existing(const DM &dmda, const Vec &srcvec, Vec &tgtvec, integ } // Release pointers and allow petsc to cleanup perr = DMDAVecRestoreArray(dmda, srcvec, &img_array); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); perr = DMDAVecRestoreArray(dmda, tgtvec, &grad_array); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); } diff --git a/src/hdfwriter.cpp b/src/hdfwriter.cpp index 83f53c1..4c8cf95 100644 --- a/src/hdfwriter.cpp +++ b/src/hdfwriter.cpp @@ -58,10 +58,10 @@ void HDFWriter::write_3d_dataset_parallel(uinteger ndim, const std::vectordmda(), &offset[0], &offset[1], &offset[2], &shape[0], &shape[1], &shape[2]); - CHKERRABORT(comm, perr); + CHKERRXX(perr); // std::transform(shape.cbegin(), shape.cend(), offset.cbegin(), shape.begin(), std::minus<>()); floating ***vecptr(nullptr); perr = DMDAVecGetArray(*new_image->dmda(), *new_image->global_vec(), &vecptr); - CHKERRABORT(comm, perr); + CHKERRXX(perr); loader->copy_scaled_chunk(vecptr, shape, offset); perr = DMDAVecRestoreArray(*new_image->dmda(), *new_image->global_vec(), &vecptr); - CHKERRABORT(comm, perr); + CHKERRXX(perr); new_image->normalize(); diff --git a/src/imagebase.cpp b/src/imagebase.cpp index c02bc41..340013b 100644 --- a/src/imagebase.cpp +++ b/src/imagebase.cpp @@ -66,26 +66,26 @@ const floating* ImageBase::get_raw_data_ro() const { const floating* ptr; PetscErrorCode perr = VecGetArrayRead(*m_globalvec, &ptr); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); return ptr; } void ImageBase::release_raw_data_ro(const floating*& ptr) const { PetscErrorCode perr = VecRestoreArrayRead(*m_globalvec, &ptr); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } Vec_unique ImageBase::get_raw_data_row_major() const { Vec_unique rmlocalpart = create_unique_vec(); PetscErrorCode perr = VecDuplicate(*m_globalvec, rmlocalpart.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); intvector locs(3, 0); intvector widths(3, 0); perr = DMDAGetCorners(*m_dmda, &locs[0], &locs[1], &locs[2], &widths[0], &widths[1], &widths[2]); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); integer ownedlo; perr = VecGetOwnershipRange(*m_globalvec, &ownedlo, nullptr); @@ -100,18 +100,18 @@ Vec_unique ImageBase::get_raw_data_row_major() const IS_unique src_is = create_unique_is(); perr = ISCreateStride(m_comm, localsize, ownedlo, 1, src_is.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); IS_unique tgt_is = create_unique_is(); perr = ISCreateGeneral(m_comm, localsize, cmidxn.data(), PETSC_USE_POINTER, tgt_is.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); VecScatter_unique sct = create_unique_vecscatter(); perr = VecScatterCreate(*m_globalvec, *src_is, *rmlocalpart, *tgt_is, sct.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecScatterBegin(*sct, *m_globalvec, *rmlocalpart, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecScatterEnd(*sct, *m_globalvec, *rmlocalpart, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); return rmlocalpart; } @@ -119,13 +119,13 @@ Vec_unique ImageBase::get_raw_data_row_major() const floating ImageBase::masked_normalize(const Mask& mask) { Vec_unique tmp = create_unique_vec(); - PetscErrorCode perr = VecDuplicate(*m_globalvec, tmp.get());CHKERRABORT(m_comm, perr); - perr = VecPointwiseMult(*tmp, *mask.global_vec(), *m_globalvec);CHKERRABORT(m_comm, perr); + PetscErrorCode perr = VecDuplicate(*m_globalvec, tmp.get());CHKERRXX(perr); + perr = VecPointwiseMult(*tmp, *mask.global_vec(), *m_globalvec);CHKERRXX(perr); floating norm; - perr = VecSum(*tmp, &norm);CHKERRABORT(m_comm, perr); + perr = VecSum(*tmp, &norm);CHKERRXX(perr); norm = mask.npoints() / norm; - perr = VecScale(*m_globalvec, norm);CHKERRABORT(m_comm, perr); + perr = VecScale(*m_globalvec, norm);CHKERRXX(perr); return norm; } @@ -133,7 +133,7 @@ floating ImageBase::masked_normalize(const Mask& mask) void ImageBase::copy_data(const ImageBase &img) { PetscErrorCode perr = VecCopy(*img.global_vec(), *m_globalvec); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); update_local_from_global(); } @@ -142,7 +142,7 @@ Vec_unique ImageBase::gradient(integer dim) // New global vec must be a duplicate of image global Vec_shared grad = create_shared_vec(); PetscErrorCode perr = VecDuplicate(*(m_globalvec), grad.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // Ensure we have up to date ghost cells DMGlobalToLocalBegin(*m_dmda, *m_globalvec, INSERT_VALUES, *m_localvec); @@ -168,10 +168,10 @@ floating ImageBase::normalize() { floating norm; PetscErrorCode perr = VecMax(*m_globalvec, nullptr, &norm); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); norm = 1.0 / norm; perr = VecScale(*m_globalvec, norm); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); return norm; } @@ -206,10 +206,10 @@ void ImageBase::initialize_dmda() dof_per_node, stencil_width, // dof per node, stencil size nullptr, nullptr, nullptr, // partition sizes nullptr -> petsc chooses m_dmda.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = DMSetUp(*(m_dmda)); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } void ImageBase::initialize_vectors() @@ -218,19 +218,19 @@ void ImageBase::initialize_vectors() m_localvec = create_shared_vec(); m_globalvec = create_shared_vec(); perr = DMCreateGlobalVector(*m_dmda, m_globalvec.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); debug_creation(*m_globalvec, std::string("image_global_") + std::to_string(instance_id)); perr = DMCreateLocalVector(*m_dmda, m_localvec.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); debug_creation(*m_localvec, std::string("image_local_") + std::to_string(instance_id)); } void ImageBase::update_local_from_global() { PetscErrorCode perr = DMGlobalToLocalBegin(*m_dmda, *m_globalvec, INSERT_VALUES, *m_localvec); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); perr = DMGlobalToLocalEnd(*m_dmda, *m_globalvec, INSERT_VALUES, *m_localvec); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); } Vec_unique ImageBase::get_raw_data_natural() const @@ -259,19 +259,19 @@ floating ImageBase::mutual_information(const ImageBase &other) integer img_localsize; PetscErrorCode perr = VecGetLocalSize(*m_globalvec, &img_localsize); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); floating max1, max2; - perr = VecMax(*m_globalvec, nullptr, &max1);CHKERRABORT(m_comm, perr); - perr = VecMax(*other.global_vec(), nullptr, &max2);CHKERRABORT(m_comm, perr); + perr = VecMax(*m_globalvec, nullptr, &max1);CHKERRXX(perr); + perr = VecMax(*other.global_vec(), nullptr, &max2);CHKERRXX(perr); floating max = std::max(max1, max2); // Only need RO data from PETSc vecs floating const *x_data, *y_data; perr = VecGetArrayRead(*m_globalvec, &x_data); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecGetArrayRead(*other.global_vec(), &y_data); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); for (integer idx = 0; idx < img_localsize; idx++) { diff --git a/src/imagebase.hpp b/src/imagebase.hpp index 401e29a..606b0b4 100644 --- a/src/imagebase.hpp +++ b/src/imagebase.hpp @@ -97,7 +97,7 @@ std::vector ImageBase::mpi_get_chunksize() const intvector sizes(3, 0); PetscErrorCode perr = DMDAGetCorners(*m_dmda, nullptr, nullptr, nullptr, &sizes[0], &sizes[1], &sizes[2]); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); std::vector out(sizes.begin(), sizes.end()); @@ -113,7 +113,7 @@ std::vector ImageBase::mpi_get_offset() const intvector offsets(3, 0); PetscErrorCode perr = DMDAGetCorners(*m_dmda, &offsets[0], &offsets[1], &offsets[2], nullptr, nullptr, nullptr); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); std::vector out(offsets.begin(), offsets.end()); diff --git a/src/laplacian.cpp b/src/laplacian.cpp index 7700d0c..e88112f 100644 --- a/src/laplacian.cpp +++ b/src/laplacian.cpp @@ -94,6 +94,6 @@ Mat_unique build_laplacian_matrix( comm, idxn.size() - 1, PETSC_DECIDE, matsize, matsize, idxn.data(), idxm.data(), mdat.data(), lapl_mat.get()); debug_creation(*lapl_mat, "laplacian"); - CHKERRABORT(comm, perr); + CHKERRXX(perr); return lapl_mat; } diff --git a/src/map.cpp b/src/map.cpp index 59c1da4..7417256 100644 --- a/src/map.cpp +++ b/src/map.cpp @@ -53,7 +53,7 @@ void Map::apply_mask_to_basis() // First stack basis to depth of map Vec_unique stacked_basis = create_unique_vec(); PetscErrorCode perr = MatCreateVecs(*m_basis, nullptr, stacked_basis.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); repeat_stack(*m_mask.global_vec(), *stacked_basis); @@ -64,7 +64,7 @@ void Map::apply_mask_to_basis() // Mask applies directly to image side of basis perr = MatDiagonalScale(*m_basis, *stacked_basis, *map_side); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } Vec_unique Map::calculate_map_mask(Vec& stacked_mask ) @@ -72,16 +72,16 @@ Vec_unique Map::calculate_map_mask(Vec& stacked_mask ) // Create map mask from image mask and basis Vec_unique map_mask = create_unique_vec(); PetscErrorCode perr = MatCreateVecs(*m_basis, map_mask.get(), nullptr); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = MatMultTranspose(*m_basis, stacked_mask, *map_mask); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // Dilate to include surrounding nodes // First need 1 copy in global layout Vec_unique map_mask_global = create_unique_vec(); perr = DMCreateGlobalVector(*map_dmda, map_mask_global.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); copy_nth_from_stack_nat_to_petsc(*map_mask_global, *map_mask, *map_dmda, 0); // Now dilate @@ -97,7 +97,7 @@ Vec_unique Map::calculate_map_mask(Vec& stacked_mask ) void Map::update(const Vec& delta_vec) { PetscErrorCode perr = VecAXPY(*m_displacements, 1, delta_vec); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } std::unique_ptr Map::interpolate(const floatvector& new_spacing) @@ -117,7 +117,7 @@ std::unique_ptr Map::interpolate(const floatvector& new_spacing) m_comm, map_shape, new_map->map_shape, scalings, offsets, m_ndim, m_ndim + 1); PetscErrorCode perr = MatMult(*interp, *m_displacements, *new_map->m_displacements); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); return new_map; } @@ -142,10 +142,10 @@ void Map::initialize_dmda() const dof_per_node, stencil_width, // dof per node, stencil size nullptr, nullptr, nullptr, // partition sizes nullptr -> petsc chooses map_dmda.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = DMSetUp(*(map_dmda)); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } void Map::alloc_displacements() { MatCreateVecs(*m_basis, m_displacements.get(), nullptr); } @@ -191,7 +191,7 @@ std::unique_ptr Map::warp(const Image& image, WorkSpace& wksp) // interpolate map to image nodes with basis PetscErrorCode perr = MatMult(*m_basis, *m_displacements, *wksp.m_stacktmp); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); wksp.scatter_stacked_to_grads_noreorder(); // build warp matrix @@ -206,27 +206,27 @@ std::unique_ptr Map::warp(const Image& image, WorkSpace& wksp) // first need image in natural ordering Vec_unique src_nat = create_unique_vec(); perr = DMDACreateNaturalVector(*image.dmda(), src_nat.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); debug_creation(*src_nat, "Vec_source_natural"); perr = DMDAGlobalToNaturalBegin(*image.dmda(), *image.global_vec(), INSERT_VALUES, *src_nat); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = DMDAGlobalToNaturalEnd(*image.dmda(), *image.global_vec(), INSERT_VALUES, *src_nat); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // do mult Vec_unique tgt_nat = create_unique_vec(); perr = DMDACreateNaturalVector(*image.dmda(), tgt_nat.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); debug_creation(*tgt_nat, "Vec_target_natural"); perr = MatMult(*warp, *src_nat, *tgt_nat); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // create new image and insert data in petsc ordering std::unique_ptr new_image = Image::duplicate(image); perr = DMDANaturalToGlobalBegin(*image.dmda(), *tgt_nat, INSERT_VALUES, *new_image->global_vec()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = DMDANaturalToGlobalEnd(*image.dmda(), *tgt_nat, INSERT_VALUES, *new_image->global_vec()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); return new_image; } @@ -238,7 +238,7 @@ std::pair Map::get_dmda_local_extents() const intvector widths(3, 0); PetscErrorCode perr = DMDAGetCorners(*map_dmda, &locs[0], &locs[1], &locs[2], &widths[0], &widths[1], &widths[2]); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); return std::make_pair(locs, widths); } @@ -253,7 +253,7 @@ Vec_unique Map::get_raw_data_row_major(uinteger dim) const intvector widths(3, 0); PetscErrorCode perr = DMDAGetCorners(*map_dmda, &locs[0], &locs[1], &locs[2], &widths[0], &widths[1], &widths[2]); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); integer startelem, localsize; perr = VecGetOwnershipRange(*tmp_vec, &startelem, &localsize); localsize -= startelem; @@ -261,7 +261,7 @@ Vec_unique Map::get_raw_data_row_major(uinteger dim) const // source is simple stride IS_unique src_is = create_unique_is(); perr = ISCreateStride(m_comm, localsize, startelem, 1, src_is.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // do rowmaj -> colmaj scatter as in Image intvector cmidxn(localsize); @@ -272,19 +272,19 @@ Vec_unique Map::get_raw_data_row_major(uinteger dim) const }); IS_unique tgt_is = create_unique_is(); perr = ISCreateGeneral(m_comm, localsize, cmidxn.data(), PETSC_USE_POINTER, tgt_is.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // now do scatter Vec_unique cmaj_vec(create_unique_vec()); perr = VecDuplicate(*tmp_vec, cmaj_vec.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); VecScatter_unique sct(create_unique_vecscatter()); perr = VecScatterCreate(*tmp_vec, *src_is, *cmaj_vec, *tgt_is, sct.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecScatterBegin(*sct, *tmp_vec, *cmaj_vec, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecScatterEnd(*sct, *tmp_vec, *cmaj_vec, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); return cmaj_vec; } @@ -300,12 +300,12 @@ Vec_unique Map::get_single_dim_petsc(uinteger dim) const // Allocate temp vec Vec_unique tmp_data = create_unique_vec(); PetscErrorCode perr = DMCreateGlobalVector(*map_dmda, tmp_data.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); AO ao_petsctonat; // N.B this is not going to be a leak, we are just borrowing a Petsc managed // obj. perr = DMDAGetAO(*map_dmda, &ao_petsctonat); // Destroying this would break the dm - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // Target range is owned part of new vec integer startelem, localsize; @@ -313,9 +313,9 @@ Vec_unique Map::get_single_dim_petsc(uinteger dim) const localsize -= startelem; IS_unique tgt_is(create_unique_is()); perr = ISCreateStride(m_comm, localsize, startelem, 1, tgt_is.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = AOPetscToApplicationIS(ao_petsctonat, *tgt_is); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); <<<<<<< HEAD // Source range is equivalent range offset to dimension @@ -325,16 +325,16 @@ Vec_unique Map::get_single_dim_petsc(uinteger dim) const >>>>>>> develop IS_unique src_is(create_unique_is()); perr = ISCreateStride(m_comm, localsize, startelem + offset, 1, src_is.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); VecScatter_unique sct = create_unique_vecscatter(); perr = VecScatterCreate(*m_displacements, *src_is, *tmp_data, *tgt_is, sct.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecScatterBegin(*sct, *m_displacements, *tmp_data, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecScatterEnd(*sct, *m_displacements, *tmp_data, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); return tmp_data; }*/ @@ -351,7 +351,7 @@ Vec_unique Map::get_single_dim_petsc(uinteger dim) const // Allocate temp vec Vec_unique tmp_data(create_unique_vec()); PetscErrorCode perr = DMCreateGlobalVector(*map_dmda, tmp_data.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); DMDANaturalToGlobalBegin(*map_dmda, *nat_data, INSERT_VALUES, *tmp_data); DMDANaturalToGlobalEnd(*map_dmda, *nat_data, INSERT_VALUES, *tmp_data); @@ -369,7 +369,7 @@ Vec_unique Map::get_single_dim_natural(uinteger dim) const // Allocate temp vec Vec_unique tmp_data = create_unique_vec(); PetscErrorCode perr = DMDACreateNaturalVector(*map_dmda, tmp_data.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // Target range is owned part of new vec integer startelem, localsize; @@ -377,7 +377,7 @@ Vec_unique Map::get_single_dim_natural(uinteger dim) const localsize -= startelem; IS_unique tgt_is(create_unique_is()); perr = ISCreateStride(m_comm, localsize, startelem, 1, tgt_is.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // Source range is owned part of vec offset by dimension length integer offset = dim * this->size(); @@ -386,12 +386,12 @@ Vec_unique Map::get_single_dim_natural(uinteger dim) const VecScatter_unique sct = create_unique_vecscatter(); perr = VecScatterCreate(*m_displacements, *src_is, *tmp_data, *tgt_is, sct.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecScatterBegin(*sct, *m_displacements, *tmp_data, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecScatterEnd(*sct, *m_displacements, *tmp_data, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); return tmp_data; } @@ -414,12 +414,12 @@ void Map::calculate_laplacian() { integer startrow, endrow; PetscErrorCode perr = VecGetOwnershipRange(*m_displacements, &startrow, &endrow); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); Mat_unique lapl = build_laplacian_matrix(m_comm, map_shape, startrow, endrow, m_ndim + 1); perr = MatTransposeMatMult(*lapl, *lapl, MAT_INITIAL_MATRIX, PETSC_DEFAULT, m_lapl.get()); debug_creation(*m_lapl, "Mat_l_squared"); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } std::pair Map::get_displacement_ownershiprange() const @@ -427,7 +427,7 @@ std::pair Map::get_displacement_ownershiprange() const std::pair range; PetscErrorCode perr = VecGetOwnershipRange(*m_displacements, &range.first, &range.second); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); return range; } @@ -436,14 +436,14 @@ const floating* Map::get_raw_data_ro() const { const floating* ptr; PetscErrorCode perr = VecGetArrayRead(*m_displacements, &ptr); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); return ptr; } void Map::release_raw_data_ro(const floating*& ptr) const { PetscErrorCode perr = VecRestoreArrayRead(*m_displacements, &ptr); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } floatvector Map::low_corner() const diff --git a/src/mask.cpp b/src/mask.cpp index 485d37d..045686f 100644 --- a/src/mask.cpp +++ b/src/mask.cpp @@ -74,15 +74,15 @@ Mask::load_file(const std::string& path, const ImageBase* existing, MPI_Comm com intvector shape(3, 0), offset(3, 0); PetscErrorCode perr = DMDAGetCorners( *new_mask->dmda(), &offset[0], &offset[1], &offset[2], &shape[0], &shape[1], &shape[2]); - CHKERRABORT(comm, perr); + CHKERRXX(perr); // std::transform(shape.cbegin(), shape.cend(), offset.cbegin(), shape.begin(), std::minus<>()); floating*** vecptr(nullptr); perr = DMDAVecGetArray(*new_mask->dmda(), *new_mask->global_vec(), &vecptr); - CHKERRABORT(comm, perr); + CHKERRXX(perr); loader->copy_scaled_chunk(vecptr, shape, offset); perr = DMDAVecRestoreArray(*new_mask->dmda(), *new_mask->global_vec(), &vecptr); - CHKERRABORT(comm, perr); + CHKERRXX(perr); new_mask->binarize(); @@ -93,7 +93,7 @@ integer Mask::npoints() const { floating total_points; PetscErrorCode perr = VecSum(*m_globalvec, &total_points); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); return static_cast(total_points); } diff --git a/src/oiiowriter.cpp b/src/oiiowriter.cpp index 4acddab..c32be42 100644 --- a/src/oiiowriter.cpp +++ b/src/oiiowriter.cpp @@ -47,11 +47,11 @@ std::string OIIOWriter::write_image(const Image& image) floating* pixdata; PetscErrorCode perr = VecGetArray(*imgvec, &pixdata); - CHKERRABORT(_comm, perr); + CHKERRXX(perr); img->write_image(OIIO::TypeDesc::DOUBLE, pixdata); img->close(); perr = VecRestoreArray(*imgvec, &pixdata); - CHKERRABORT(_comm, perr); + CHKERRXX(perr); //Manual ptr management if needed #if OIIO_VERSION < 10903 diff --git a/src/petsc_helpers.cpp b/src/petsc_helpers.cpp index ca709a9..bc328d3 100644 --- a/src/petsc_helpers.cpp +++ b/src/petsc_helpers.cpp @@ -28,10 +28,10 @@ floating diagonal_sum(const Mat& matrix) Vec_unique diag = create_unique_vec(); PetscErrorCode perr = MatCreateVecs(matrix, diag.get(), nullptr); - CHKERRABORT(comm, perr); + CHKERRXX(perr); perr = MatGetDiagonal(matrix, *diag); - CHKERRABORT(comm, perr); + CHKERRXX(perr); floating diagsum; perr = VecSum(*diag, &diagsum); @@ -44,9 +44,9 @@ bool vecs_equivalent(const Vec &vec1, const Vec &vec2) // Ensure two vectors have the same local size, global size. and comm. MPI_Comm comm1, comm2; PetscErrorCode perr = PetscObjectGetComm((PetscObject)vec1, &comm1); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); perr = PetscObjectGetComm((PetscObject)vec2, &comm2); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); if (comm1 != comm2) { return false; @@ -54,9 +54,9 @@ bool vecs_equivalent(const Vec &vec1, const Vec &vec2) integer globalsize1, globalsize2; perr = VecGetSize(vec1, &globalsize1); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); perr = VecGetSize(vec2, &globalsize2); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); if (globalsize1 != globalsize2) { return false; @@ -64,9 +64,9 @@ bool vecs_equivalent(const Vec &vec1, const Vec &vec2) integer localsize1, localsize2; perr = VecGetLocalSize(vec1, &localsize1); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); perr = VecGetLocalSize(vec2, &localsize2); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); if (localsize1 != localsize2) { return false; @@ -82,10 +82,10 @@ void block_precondition(const Mat& normmat, integer size, integer ndim) // Normalize luminance block of matrix to spatial blocks using diagonal norm Vec_unique diag = create_unique_vec(); PetscErrorCode perr = MatCreateVecs(normmat, diag.get(), nullptr); - CHKERRABORT(comm, perr); + CHKERRXX(perr); debug_creation(*diag, "Vec_block_normalise"); perr = MatGetDiagonal(normmat, *diag); - CHKERRABORT(comm, perr); + CHKERRXX(perr); // index where spatial dims stop and intensity dim starts integer crit_idx = size * ndim; @@ -93,7 +93,7 @@ void block_precondition(const Mat& normmat, integer size, integer ndim) // Find rank-local sums for spatial and luminance blocks integer rowstart, rowend; perr = VecGetOwnershipRange(*diag, &rowstart, &rowend); - CHKERRABORT(comm, perr); + CHKERRXX(perr); integer localsize = rowend - rowstart; floatvector norm(2, 0.0); // norm[0] is spatial, norm[1] is luminance integer spt_end = crit_idx - rowstart; @@ -101,7 +101,7 @@ void block_precondition(const Mat& normmat, integer size, integer ndim) floating* ptr; perr = VecGetArray(*diag, &ptr); - CHKERRABORT(comm, perr); + CHKERRXX(perr); for (integer idx = 0; idx < spt_end; idx++) { norm[0] += ptr[idx]; @@ -111,7 +111,7 @@ void block_precondition(const Mat& normmat, integer size, integer ndim) norm[1] += ptr[idx]; } perr = VecRestoreArray(*diag, &ptr); - CHKERRABORT(comm, perr); + CHKERRXX(perr); // MPI_AllReduce to sum over all processes MPI_Allreduce(MPI_IN_PLACE, norm.data(), 2, MPI_DOUBLE, MPI_SUM, comm); @@ -123,7 +123,7 @@ void block_precondition(const Mat& normmat, integer size, integer ndim) // reuse diag vector to hold scaling values perr = VecGetArray(*diag, &ptr); - CHKERRABORT(comm, perr); + CHKERRXX(perr); for (integer idx = 0; idx < spt_end; idx++) { ptr[idx] = 1; @@ -133,10 +133,10 @@ void block_precondition(const Mat& normmat, integer size, integer ndim) ptr[idx] = lum_scale; } perr = VecRestoreArray(*diag, &ptr); - CHKERRABORT(comm, perr); + CHKERRXX(perr); perr = MatDiagonalScale(normmat, *diag, nullptr); - CHKERRABORT(comm, perr); + CHKERRXX(perr); } floating get_condnum_by_poweriter(const Mat& matrix, floating conv_thres, integer max_iter) @@ -149,9 +149,9 @@ floating get_condnum_by_poweriter(const Mat& matrix, floating conv_thres, intege // Get smallest eigenvalue by spectral shift Mat_unique e_n_mat = create_unique_mat(); PetscErrorCode perr = MatDuplicate(matrix, MAT_DO_NOT_COPY_VALUES, e_n_mat.get()); - CHKERRABORT(comm, perr); + CHKERRXX(perr); perr = MatShift(*e_n_mat, -eigen_lo); - CHKERRABORT(comm, perr); + CHKERRXX(perr); floating eigen_hi = get_eigenvalue_by_poweriter(*e_n_mat, conv_thres, max_iter) + eigen_lo; @@ -179,37 +179,37 @@ floating get_eigenvalue_by_poweriter(const Mat& matrix, floating conv_thres, int // Initial guess at b_0 Vec_unique b = create_unique_vec(); PetscErrorCode perr = MatCreateVecs(matrix, b.get(), nullptr); - CHKERRABORT(comm, perr); + CHKERRXX(perr); perr = VecSet(*b, 1.0); - CHKERRABORT(comm, perr); + CHKERRXX(perr); Vec_unique b_next = create_unique_vec(); perr = MatCreateVecs(matrix, b_next.get(), nullptr); - CHKERRABORT(comm, perr); + CHKERRXX(perr); integer curr_iter = 0; floating delta = 2 * conv_thres; floating lambda = 1.0; floating b_dot; perr = VecDot(*b, *b, &b_dot); - CHKERRABORT(comm, perr); + CHKERRXX(perr); while (delta > conv_thres && curr_iter++ < max_iter) { perr = MatMult(matrix, *b, *b_next); - CHKERRABORT(comm, perr); + CHKERRXX(perr); floating b_bn_dot; perr = VecDot(*b, *b_next, &b_bn_dot); - CHKERRABORT(comm, perr); + CHKERRXX(perr); floating new_lambda = b_bn_dot / b_dot; delta = std::abs(new_lambda - lambda); perr = VecDot(*b_next, *b_next, &b_dot); - CHKERRABORT(comm, perr); + CHKERRXX(perr); perr = VecScale(*b_next, 1. / std::sqrt(b_dot)); - CHKERRABORT(comm, perr); + CHKERRXX(perr); b.swap(b_next); lambda = new_lambda; @@ -249,12 +249,12 @@ void repeat_stack(const Vec& subvec, const Vec& stacked) // Get extents of local data in subvector integer startelem, datasize; PetscErrorCode perr = VecGetOwnershipRange(subvec, &startelem, &datasize); - CHKERRABORT(comm, perr); + CHKERRXX(perr); datasize -= startelem; // Create IS for source indices in stacked array IS_unique src_is = create_unique_is(); perr = ISCreateStride(comm, datasize, startelem, 1, src_is.get()); - CHKERRABORT(comm, perr); + CHKERRXX(perr); integer stack_total = stacked_len / subvec_len; integer offset = 0; // Track offset @@ -263,17 +263,17 @@ void repeat_stack(const Vec& subvec, const Vec& stacked) IS_unique tgt_is = create_unique_is(); perr = ISCreateStride(comm, datasize, startelem + offset, 1, tgt_is.get()); - CHKERRABORT(comm, perr); + CHKERRXX(perr); // Create scatterer and add to array VecScatter_unique scatterer = create_unique_vecscatter(); perr = VecScatterCreate(subvec, *src_is, stacked, *tgt_is, scatterer.get()); - CHKERRABORT(comm, perr); + CHKERRXX(perr); perr = VecScatterBegin(*scatterer, subvec, stacked, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(comm, perr); + CHKERRXX(perr); perr = VecScatterEnd(*scatterer, subvec, stacked, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(comm, perr); + CHKERRXX(perr); offset += subvec_len; } @@ -304,20 +304,20 @@ void repeat_stack_petsc_to_nat(const Vec& subvec, const Vec& stacked, const DM& AO ao_petsctonat; // N.B this is not going to be a leak, we are just borrowing a Petsc managed // obj. PetscErrorCode perr = DMDAGetAO(subvec_dmda, &ao_petsctonat); // Destroying this would break the dmda - CHKERRABORT(comm, perr); + CHKERRXX(perr); // Get extents of local data in subvector integer startelem, datasize; perr = VecGetOwnershipRange(subvec, &startelem, &datasize); - CHKERRABORT(comm, perr); + CHKERRXX(perr); datasize -= startelem; // Create IS for source indices in stacked array IS_unique src_is = create_unique_is(); perr = ISCreateStride(comm, datasize, startelem, 1, src_is.get()); - CHKERRABORT(comm, perr); + CHKERRXX(perr); // Convert with AO to map from natural to petsc ordering (do here because tgt_is is offset) perr = AOApplicationToPetscIS(ao_petsctonat, *src_is); - CHKERRABORT(comm, perr); + CHKERRXX(perr); integer stack_total = stacked_len / subvec_len; integer offset = 0; // Track offset @@ -326,17 +326,17 @@ void repeat_stack_petsc_to_nat(const Vec& subvec, const Vec& stacked, const DM& IS_unique tgt_is = create_unique_is(); perr = ISCreateStride(comm, datasize, startelem + offset, 1, tgt_is.get()); - CHKERRABORT(comm, perr); + CHKERRXX(perr); // Create scatterer and add to array VecScatter_unique scatterer = create_unique_vecscatter(); perr = VecScatterCreate(subvec, *src_is, stacked, *tgt_is, scatterer.get()); - CHKERRABORT(comm, perr); + CHKERRXX(perr); perr = VecScatterBegin(*scatterer, subvec, stacked, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(comm, perr); + CHKERRXX(perr); perr = VecScatterEnd(*scatterer, subvec, stacked, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(comm, perr); + CHKERRXX(perr); offset += subvec_len; } @@ -375,36 +375,36 @@ void copy_nth_from_stack_nat_to_petsc(const Vec& subvec, const Vec& stacked, con AO ao_petsctonat; // N.B this is not going to be a leak, we are just borrowing a Petsc managed // obj. PetscErrorCode perr = DMDAGetAO(subvec_dmda, &ao_petsctonat); // Destroying this would break the dmda - CHKERRABORT(comm, perr); + CHKERRXX(perr); // Get extents of local data in subvector integer startelem, datasize; perr = VecGetOwnershipRange(subvec, &startelem, &datasize); - CHKERRABORT(comm, perr); + CHKERRXX(perr); datasize -= startelem; // Create IS for source indices in stacked array IS_unique src_is = create_unique_is(); perr = ISCreateStride(comm, datasize, startelem, 1, src_is.get()); - CHKERRABORT(comm, perr); + CHKERRXX(perr); // Convert with AO to map from natural to petsc ordering (do here because tgt_is is offset) // This direction because then we'll reverse the scatter perr = AOApplicationToPetscIS(ao_petsctonat, *src_is); - CHKERRABORT(comm, perr); + CHKERRXX(perr); integer offset = n*subvec_len; // Track offset IS_unique tgt_is = create_unique_is(); perr = ISCreateStride(comm, datasize, startelem + offset, 1, tgt_is.get()); - CHKERRABORT(comm, perr); + CHKERRXX(perr); // Create scatterer and add to array VecScatter_unique scatterer = create_unique_vecscatter(); perr = VecScatterCreate(subvec, *src_is, stacked, *tgt_is, scatterer.get()); - CHKERRABORT(comm, perr); + CHKERRXX(perr); perr = VecScatterBegin(*scatterer, stacked, subvec, INSERT_VALUES, SCATTER_REVERSE); - CHKERRABORT(comm, perr); + CHKERRXX(perr); perr = VecScatterEnd(*scatterer, stacked, subvec, INSERT_VALUES, SCATTER_REVERSE); - CHKERRABORT(comm, perr); + CHKERRXX(perr); } diff --git a/src/setup.cpp b/src/setup.cpp index 312d75a..ce02947 100644 --- a/src/setup.cpp +++ b/src/setup.cpp @@ -76,7 +76,7 @@ void pfire_setup(const std::vector& petsc_args, bool silent) char** cstr_ptr = cstrings.data(); PetscErrorCode perr = PetscInitialize(&n_cstrings, &cstr_ptr, nullptr, nullptr); - CHKERRABORT(PETSC_COMM_WORLD, perr); + CHKERRXX(perr); int rank; MPI_Comm_rank(PETSC_COMM_WORLD, &rank); diff --git a/src/workspace.cpp b/src/workspace.cpp index ab7c1be..a0005d8 100644 --- a/src/workspace.cpp +++ b/src/workspace.cpp @@ -29,21 +29,21 @@ WorkSpace::WorkSpace(const Image& image, const Map& map) Vec_unique tmp_vec = create_unique_vec(); PetscErrorCode perr = VecDuplicate(*image.global_vec(), tmp_vec.get()); debug_creation(*tmp_vec, std::string("grads_") + std::to_string(static_cast(idim))); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); m_globaltmps.push_back(std::move(tmp_vec)); } // create local vector to allow gradient calculation m_localtmp = create_unique_vec(); PetscErrorCode perr = VecDuplicate(*image.local_vec(), m_localtmp.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // create "global" vector for stack compatible with basis matrix // should be compatible with all map bases of this size m_stacktmp = create_unique_vec(); perr = MatCreateVecs(*map.basis(), nullptr, m_stacktmp.get()); debug_creation(*m_stacktmp, "workspace vector"); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); create_reordering_scatterers(); create_nonreordering_scatterers(); @@ -58,15 +58,15 @@ void WorkSpace::reallocate_ephemeral_workspace(const Map& map) m_delta = create_unique_vec(); m_rhs = create_unique_vec(); perr = VecDuplicate(*map.m_displacements, m_delta.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); debug_creation(*m_delta, std::string("solution_storage_") + std::to_string(ephemeral_count)); perr = VecSet(*m_delta, 0.); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecDuplicate(*map.m_displacements, m_rhs.get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); debug_creation(*m_delta, std::string("rhs_storage_") + std::to_string(ephemeral_count)); perr = VecSet(*m_rhs, 0.); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } void WorkSpace::scatter_stacked_to_grads() @@ -75,10 +75,10 @@ void WorkSpace::scatter_stacked_to_grads() { PetscErrorCode perr = VecScatterBegin( *m_r_scatterers[idim], *m_stacktmp, *m_globaltmps[idim], INSERT_VALUES, SCATTER_REVERSE); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecScatterEnd( *m_r_scatterers[idim], *m_stacktmp, *m_globaltmps[idim], INSERT_VALUES, SCATTER_REVERSE); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } } @@ -88,10 +88,10 @@ void WorkSpace::scatter_grads_to_stacked() { PetscErrorCode perr = VecScatterBegin( *m_r_scatterers[idim], *m_globaltmps[idim], *m_stacktmp, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecScatterEnd( *m_r_scatterers[idim], *m_globaltmps[idim], *m_stacktmp, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } } @@ -101,10 +101,10 @@ void WorkSpace::duplicate_single_grad_to_stacked(size_t idx) { PetscErrorCode perr = VecScatterBegin( *m_r_scatterers[idim], *m_globaltmps[idx], *m_stacktmp, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecScatterEnd( *m_r_scatterers[idim], *m_globaltmps[idx], *m_stacktmp, INSERT_VALUES, SCATTER_FORWARD); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } } @@ -114,10 +114,10 @@ void WorkSpace::scatter_stacked_to_grads_noreorder() { PetscErrorCode perr = VecScatterBegin( *m_nr_scatterers[idim], *m_stacktmp, *m_globaltmps[idim], INSERT_VALUES, SCATTER_REVERSE); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); perr = VecScatterEnd( *m_nr_scatterers[idim], *m_stacktmp, *m_globaltmps[idim], INSERT_VALUES, SCATTER_REVERSE); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); } } @@ -129,25 +129,25 @@ void WorkSpace::create_nonreordering_scatterers() // Get extents of local data in grad array integer startelem, datasize; PetscErrorCode perr = VecGetOwnershipRange(*pp_grad, &startelem, &datasize); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); datasize -= startelem; // Create IS for source indices in stacked array, keep in vector for memory management m_iss.push_back(create_unique_is()); IS* p_src_is = m_iss.back().get(); // grab a temp copy to avoid vector accesses below perr = ISCreateStride(m_comm, datasize, startelem, 1, p_src_is); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); m_iss.push_back(create_unique_is()); IS* p_tgt_is = m_iss.back().get(); // grab a temp copy to avoid vector accesses below perr = ISCreateStride(m_comm, datasize, startelem + offset, 1, p_tgt_is); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // Create scatterer and add to array m_nr_scatterers.push_back(create_unique_vecscatter()); perr = VecScatterCreate(*pp_grad, *p_src_is, *m_stacktmp, *p_tgt_is, m_nr_scatterers.back().get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); offset += m_size; } @@ -158,7 +158,7 @@ void WorkSpace::create_reordering_scatterers() AO ao_petsctonat; // N.B this is not going to be a leak, we are just borrowing a Petsc managed // obj. PetscErrorCode perr = DMDAGetAO(*m_dmda, &ao_petsctonat); // Destroying this would break the dmda - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); integer offset = 0; // Track offset for (auto&& pp_grad : m_globaltmps) @@ -166,28 +166,28 @@ void WorkSpace::create_reordering_scatterers() // Get extents of local data in grad array integer startelem, datasize; perr = VecGetOwnershipRange(*pp_grad, &startelem, &datasize); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); datasize -= startelem; // Create IS for source indices in stacked array, keep in vector for memory management m_iss.push_back(create_unique_is()); IS* p_src_is = m_iss.back().get(); // grab a temp copy to avoid vector accesses below perr = ISCreateStride(m_comm, datasize, startelem, 1, p_src_is); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // Convert with AO to map from petsc to natural ordering (do here because tgt_is is offset) perr = AOApplicationToPetscIS(ao_petsctonat, *m_iss.back()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); m_iss.push_back(create_unique_is()); IS* p_tgt_is = m_iss.back().get(); // grab a temp copy to avoid vector accesses below perr = ISCreateStride(m_comm, datasize, startelem + offset, 1, p_tgt_is); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); // Create scatterer and add to array m_r_scatterers.push_back(create_unique_vecscatter()); perr = VecScatterCreate(*pp_grad, *p_src_is, *m_stacktmp, *p_tgt_is, m_r_scatterers.back().get()); - CHKERRABORT(m_comm, perr); + CHKERRXX(perr); offset += m_size; }