diff --git a/example/gmres/gmres.hpp b/example/gmres/gmres.hpp index 5a7ee2c376..31b48e73a3 100644 --- a/example/gmres/gmres.hpp +++ b/example/gmres/gmres.hpp @@ -51,6 +51,7 @@ #include #include #include +#include "KokkosKernels_Error.hpp" //////////////////////////////////////////////////////////////////////////////// // libstdc++ half_t overloads @@ -141,7 +142,7 @@ GmresStats gmres( std::ostringstream os; os << "gmres: A must be a square matrix: " << "numRows: " << n << " numCols: " << A.numCols(); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } if (X.extent(0) != B.extent(0) || X.extent(0) != n) { @@ -149,7 +150,7 @@ GmresStats gmres( os << "gmres: Dimensions of A, X, and B do not match: " << "A: " << n << " x " << n << ", X: " << X.extent(0) << "x 1, B: " << B.extent(0) << " x 1"; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Check parameter validity: if (m <= 0) { diff --git a/src/batched/dense/KokkosBatched_Gemm_Decl.hpp b/src/batched/dense/KokkosBatched_Gemm_Decl.hpp index 9b1eb18e1f..83a51ea0f3 100644 --- a/src/batched/dense/KokkosBatched_Gemm_Decl.hpp +++ b/src/batched/dense/KokkosBatched_Gemm_Decl.hpp @@ -48,6 +48,7 @@ // Includes for non-functor-level routines #include #include +#include namespace KokkosBatched { /********************* BEGIN functor-level routines *********************/ @@ -357,7 +358,7 @@ int BatchedGemm(BatchedGemmHandleType *const handle, const ScalarType alpha, os << "KokkosBatched::BatchedGemm does not support kernelAlgoType = " << std::to_string(handle->get_kernel_algo_type()) << " with SIMD views." << std::endl; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); break; } } else { @@ -436,7 +437,7 @@ int BatchedGemm(BatchedGemmHandleType *const handle, const ScalarType alpha, << std::to_string(handle->get_kernel_algo_type()) << " when c_m(" << std::to_string(c_m) << ") != c_n(" << std::to_string(c_n) << ")" << std::endl; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Select optimal resultsPerThread param for BatchedSerialGemm @@ -577,7 +578,7 @@ int BatchedGemm(BatchedGemmHandleType *const handle, const ScalarType alpha, std::ostringstream os; os << "KokkosBatched::BatchedGemm does not support kernelAlgoType = " << std::to_string(handle->get_kernel_algo_type()) << "." << std::endl; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); break; } return ret; diff --git a/src/batched/dense/impl/KokkosBatched_Gemm_DblBuf_Impl.hpp b/src/batched/dense/impl/KokkosBatched_Gemm_DblBuf_Impl.hpp index 57cb31e764..3817c8f38d 100644 --- a/src/batched/dense/impl/KokkosBatched_Gemm_DblBuf_Impl.hpp +++ b/src/batched/dense/impl/KokkosBatched_Gemm_DblBuf_Impl.hpp @@ -43,6 +43,7 @@ #define __KOKKOSBATCHED_GEMM_DBLBUF_IMPL_HPP__ #include "KokkosBatched_Util.hpp" +#include "KokkosKernels_Error.hpp" namespace KokkosBatched { /********************* BEGIN functor-level routines *********************/ @@ -160,7 +161,7 @@ class BatchedDblBufGemm { << " does not support team_size > " << std::to_string(max_team_size) << "." << std::endl << " The tile dimensions must be adjusted." << std::endl; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } const int max_vector_len = @@ -172,7 +173,7 @@ class BatchedDblBufGemm { << " does not support vector_len > " << std::to_string(max_vector_len) << "." << std::endl << " The tile dimensions must be adjusted." << std::endl; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } if (__handle->enableDebug) { diff --git a/src/blas/KokkosBlas1_abs.hpp b/src/blas/KokkosBlas1_abs.hpp index fb2a4976ed..81b5b5bfff 100644 --- a/src/blas/KokkosBlas1_abs.hpp +++ b/src/blas/KokkosBlas1_abs.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -85,7 +86,7 @@ void abs(const RMV& R, const XMV& X) { os << "KokkosBlas::abs (MV): Dimensions of R and X do not match: " << "R: " << R.extent(0) << " x " << R.extent(1) << ", X: " << X.extent(0) << " x " << X.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Create unmanaged versions of the input Views. RMV and XMV may be diff --git a/src/blas/KokkosBlas1_axpby.hpp b/src/blas/KokkosBlas1_axpby.hpp index 4fb4660112..cae0cc7102 100644 --- a/src/blas/KokkosBlas1_axpby.hpp +++ b/src/blas/KokkosBlas1_axpby.hpp @@ -47,6 +47,7 @@ #include #include +#include // axpby() accepts both scalar coefficients a and b, and vector // coefficients (apply one for each column of the input multivectors). @@ -82,7 +83,7 @@ void axpby(const AV& a, const XMV& X, const BV& b, const YMV& Y) { os << "KokkosBlas::axpby: Dimensions of X and Y do not match: " << "X: " << X.extent(0) << " x " << X.extent(1) << ", Y: " << Y.extent(0) << " x " << Y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } using UnifiedXLayout = diff --git a/src/blas/KokkosBlas1_dot.hpp b/src/blas/KokkosBlas1_dot.hpp index 9cc611d87c..bd1de1d35a 100644 --- a/src/blas/KokkosBlas1_dot.hpp +++ b/src/blas/KokkosBlas1_dot.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -79,7 +80,7 @@ dot(const XVector& x, const YVector& y) { os << "KokkosBlas::dot: Dimensions do not match: " << ", x: " << x.extent(0) << " x 1" << ", y: " << y.extent(0) << " x 1"; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } typedef Kokkos::View< @@ -209,7 +210,7 @@ void dot(const RV& R, const XMV& X, const YMV& Y, } os << "X: " << X.extent(0) << " x " << X.extent(1) << ", Y: " << Y.extent(0) << " x " << Y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Create unmanaged versions of the input Views. diff --git a/src/blas/KokkosBlas1_iamax.hpp b/src/blas/KokkosBlas1_iamax.hpp index 674e04d00d..70363fc340 100644 --- a/src/blas/KokkosBlas1_iamax.hpp +++ b/src/blas/KokkosBlas1_iamax.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -131,7 +132,7 @@ void iamax(const RV& R, const XMV& X, os << "KokkosBlas::iamax (MV): Dimensions of R and X do not match: " << "R: " << R.extent(0) << ", X: " << X.extent(0) << " x " << X.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } using UnifiedXLayout = diff --git a/src/blas/KokkosBlas1_mult.hpp b/src/blas/KokkosBlas1_mult.hpp index 8e1cf1a4ab..f99bc067a9 100644 --- a/src/blas/KokkosBlas1_mult.hpp +++ b/src/blas/KokkosBlas1_mult.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -78,7 +79,7 @@ void mult(typename YMV::const_value_type& gamma, const YMV& Y, << "Y: " << Y.extent(0) << " x " << Y.extent(1) << ", A: " << A.extent(0) << " x " << A.extent(0) << ", X: " << X.extent(0) << " x " << X.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } using YUnifiedLayout = diff --git a/src/blas/KokkosBlas1_nrm1.hpp b/src/blas/KokkosBlas1_nrm1.hpp index c890219168..fb453a7024 100644 --- a/src/blas/KokkosBlas1_nrm1.hpp +++ b/src/blas/KokkosBlas1_nrm1.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -130,7 +131,7 @@ void nrm1(const RV& R, const XMV& X, os << "KokkosBlas::nrm1 (MV): Dimensions of R and X do not match: " << "R: " << R.extent(0) << ", X: " << X.extent(0) << " x " << X.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } using UnifiedXLayout = diff --git a/src/blas/KokkosBlas1_nrm2.hpp b/src/blas/KokkosBlas1_nrm2.hpp index 7f24939d05..3a10e48a4d 100644 --- a/src/blas/KokkosBlas1_nrm2.hpp +++ b/src/blas/KokkosBlas1_nrm2.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -130,7 +131,7 @@ void nrm2(const RV& R, const XMV& X, os << "KokkosBlas::nrm2 (MV): Dimensions of R and X do not match: " << "R: " << R.extent(0) << ", X: " << X.extent(0) << " x " << X.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } using UnifiedXLayout = diff --git a/src/blas/KokkosBlas1_nrm2_squared.hpp b/src/blas/KokkosBlas1_nrm2_squared.hpp index 0c0fb1ec78..e4054e944a 100644 --- a/src/blas/KokkosBlas1_nrm2_squared.hpp +++ b/src/blas/KokkosBlas1_nrm2_squared.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -131,7 +132,7 @@ void nrm2_squared( os << "KokkosBlas::nrm2 (MV): Dimensions of R and X do not match: " << "R: " << R.extent(0) << ", X: " << X.extent(0) << " x " << X.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } using UnifiedXLayout = diff --git a/src/blas/KokkosBlas1_nrm2w.hpp b/src/blas/KokkosBlas1_nrm2w.hpp index b5d3f5b9b5..981897d9ae 100644 --- a/src/blas/KokkosBlas1_nrm2w.hpp +++ b/src/blas/KokkosBlas1_nrm2w.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -130,7 +131,7 @@ void nrm2w(const RV& R, const XMV& X, const XMV& W, os << "KokkosBlas::nrm2w (MV): Dimensions of R and X do not match: " << "R: " << R.extent(0) << ", X: " << X.extent(0) << " x " << X.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Create unmanaged versions of the input Views. RV and XMV may be diff --git a/src/blas/KokkosBlas1_nrm2w_squared.hpp b/src/blas/KokkosBlas1_nrm2w_squared.hpp index 3e7b754337..2ab07af0c5 100644 --- a/src/blas/KokkosBlas1_nrm2w_squared.hpp +++ b/src/blas/KokkosBlas1_nrm2w_squared.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -131,7 +132,7 @@ void nrm2w_squared( os << "KokkosBlas::nrm2w (MV): Dimensions of R and X do not match: " << "R: " << R.extent(0) << ", X: " << X.extent(0) << " x " << X.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Create unmanaged versions of the input Views. RV and XMV may be diff --git a/src/blas/KokkosBlas1_nrminf.hpp b/src/blas/KokkosBlas1_nrminf.hpp index d3e29177a3..915c085376 100644 --- a/src/blas/KokkosBlas1_nrminf.hpp +++ b/src/blas/KokkosBlas1_nrminf.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -131,7 +132,7 @@ void nrminf( os << "KokkosBlas::nrminf (MV): Dimensions of R and X do not match: " << "R: " << R.extent(0) << ", X: " << X.extent(0) << " x " << X.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } using UnifiedXLayout = diff --git a/src/blas/KokkosBlas1_reciprocal.hpp b/src/blas/KokkosBlas1_reciprocal.hpp index 4ec55ac1a7..1d185d73da 100644 --- a/src/blas/KokkosBlas1_reciprocal.hpp +++ b/src/blas/KokkosBlas1_reciprocal.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -85,7 +86,7 @@ void reciprocal(const RMV& R, const XMV& X) { os << "KokkosBlas::reciprocal (MV): Dimensions of R and X do not match: " << "R: " << R.extent(0) << " x " << R.extent(1) << ", X: " << X.extent(0) << " x " << X.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Create unmanaged versions of the input Views. RMV and XMV may be diff --git a/src/blas/KokkosBlas1_scal.hpp b/src/blas/KokkosBlas1_scal.hpp index 460d0d1ee8..2fc4f92f58 100644 --- a/src/blas/KokkosBlas1_scal.hpp +++ b/src/blas/KokkosBlas1_scal.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -76,7 +77,7 @@ void scal(const RMV& R, const AV& a, const XMV& X) { os << "KokkosBlas::scal: Dimensions of R and X do not match: " << "R: " << R.extent(0) << " x " << R.extent(1) << ", X: " << X.extent(0) << " x " << X.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } using UnifiedRLayout = diff --git a/src/blas/KokkosBlas1_sum.hpp b/src/blas/KokkosBlas1_sum.hpp index e140c3f015..546b34440a 100644 --- a/src/blas/KokkosBlas1_sum.hpp +++ b/src/blas/KokkosBlas1_sum.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -120,7 +121,7 @@ void sum(const RV& R, const XMV& X, os << "KokkosBlas::sum (MV): Dimensions of R and X do not match: " << "R: " << R.extent(0) << ", X: " << X.extent(0) << " x " << X.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } using UnifiedXLayout = diff --git a/src/blas/KokkosBlas1_update.hpp b/src/blas/KokkosBlas1_update.hpp index d2d86e5464..cb934fd17b 100644 --- a/src/blas/KokkosBlas1_update.hpp +++ b/src/blas/KokkosBlas1_update.hpp @@ -47,6 +47,7 @@ #include #include +#include namespace KokkosBlas { @@ -95,7 +96,7 @@ void update(const typename XMV::non_const_value_type& alpha, const XMV& X, << "Z: " << Z.extent(0) << " x " << Z.extent(1) << ", X: " << X.extent(0) << " x " << X.extent(1) << ", Y: " << Y.extent(0) << " x " << Y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Create unmanaged versions of the input Views. XMV, YMV, and ZMV diff --git a/src/blas/KokkosBlas2_gemv.hpp b/src/blas/KokkosBlas2_gemv.hpp index 8581b413df..5c37b74c9b 100644 --- a/src/blas/KokkosBlas2_gemv.hpp +++ b/src/blas/KokkosBlas2_gemv.hpp @@ -50,6 +50,7 @@ #include #include +#include #include #include // requires C++11, but so does Kokkos @@ -99,7 +100,7 @@ void gemv(const typename AViewType::execution_space& space, const char trans[], os << "KokkosBlas::gemv: Dimensions of A, x, and y do not match: " << "A: " << A.extent(0) << " x " << A.extent(1) << ", x: " << x.extent(0) << ", y: " << y.extent(0); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else if (trans[0] == 'T' || trans[0] == 't' || trans[0] == 'C' || trans[0] == 'c' || trans[0] == 'H' || trans[0] == 'h') { @@ -108,7 +109,7 @@ void gemv(const typename AViewType::execution_space& space, const char trans[], os << "KokkosBlas::dot: Dimensions of A, x, and y do not match: " << "A: " << A.extent(0) << " x " << A.extent(1) << ", x: " << x.extent(0) << ", y: " << y.extent(0); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else { std::ostringstream os; @@ -116,7 +117,7 @@ void gemv(const typename AViewType::execution_space& space, const char trans[], << "'. Valid values " "include 'N' (No transpose), 'T' (Transpose), and 'C' (Conjugate " "transpose)."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } using ALayout = typename AViewType::array_layout; diff --git a/src/blas/KokkosBlas3_gemm.hpp b/src/blas/KokkosBlas3_gemm.hpp index d1ebb0497b..3d36ad86c9 100644 --- a/src/blas/KokkosBlas3_gemm.hpp +++ b/src/blas/KokkosBlas3_gemm.hpp @@ -51,6 +51,7 @@ #include #include #include +#include #include #include @@ -167,7 +168,7 @@ void gemm(const typename CViewType::execution_space& space, const char transA[], << "Valid values include 'N' or 'n' (No transpose), 'T' or 't' " "(Transpose), " "and 'C' or 'c' (Conjugate transpose)."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Check compatibility of dimensions at run time. @@ -189,7 +190,7 @@ void gemm(const typename CViewType::execution_space& space, const char transA[], << "transA: " << transA[0] << " transB: " << transB[0] << " A: " << A.extent(0) << " x " << A.extent(1) << " B: " << B.extent(0) << " x " << B.extent(1) << " C: " << C.extent(0) << " x " << C.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } #endif // KOKKOSKERNELS_DEBUG_LEVEL > 0 diff --git a/src/blas/KokkosBlas3_trmm.hpp b/src/blas/KokkosBlas3_trmm.hpp index 914e5e4668..b760f5d41e 100644 --- a/src/blas/KokkosBlas3_trmm.hpp +++ b/src/blas/KokkosBlas3_trmm.hpp @@ -49,6 +49,7 @@ #include "KokkosKernels_Macros.hpp" #include "KokkosBlas3_trmm_spec.hpp" #include "KokkosKernels_helpers.hpp" +#include "KokkosKernels_Error.hpp" #include #include @@ -111,14 +112,14 @@ void trmm(const char side[], const char uplo[], const char trans[], os << "KokkosBlas::trmm: side = '" << side[0] << "'. " << "Valid values include 'L' or 'l' (A is on the left of X), " "'R' or 'r' (A is on the right of X)."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } if (!valid_uplo) { std::ostringstream os; os << "KokkosBlas::trmm: uplo = '" << uplo[0] << "'. " << "Valid values include 'U' or 'u' (A is upper triangular), " "'L' or 'l' (A is lower triangular)."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } if (!valid_trans) { std::ostringstream os; @@ -126,7 +127,7 @@ void trmm(const char side[], const char uplo[], const char trans[], << "Valid values include 'N' or 'n' (No transpose), 'T' or 't' " "(Transpose), " "and 'C' or 'c' (Conjugate transpose)."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } if (!valid_diag) { std::ostringstream os; @@ -134,7 +135,7 @@ void trmm(const char side[], const char uplo[], const char trans[], << "Valid values include 'U' or 'u' (the diagonal of A is assumed to be " "unit), " "'N' or 'n' (the diagonal of A is assumed to be non-unit)."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } bool is_A_lower_triangle = (side[0] == 'L' || side[0] == 'l'); @@ -154,7 +155,7 @@ void trmm(const char side[], const char uplo[], const char trans[], os << "KokkosBlas::trmm: Dimensions of A and B do not match: " << "side: " << side[0] << " A: " << A.extent(0) << " x " << A.extent(1) << " B: " << B.extent(0) << " x " << B.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Create A matrix view type alias diff --git a/src/blas/KokkosBlas3_trsm.hpp b/src/blas/KokkosBlas3_trsm.hpp index c909388913..1e73d92049 100644 --- a/src/blas/KokkosBlas3_trsm.hpp +++ b/src/blas/KokkosBlas3_trsm.hpp @@ -49,6 +49,7 @@ #include "KokkosKernels_Macros.hpp" #include "KokkosBlas3_trsm_spec.hpp" #include "KokkosKernels_helpers.hpp" +#include "KokkosKernels_Error.hpp" #include #include @@ -108,14 +109,14 @@ void trsm(const char side[], const char uplo[], const char trans[], os << "KokkosBlas::trsm: side = '" << side[0] << "'. " << "Valid values include 'L' or 'l' (A is on the left of X), " "'R' or 'r' (A is on the right of X)."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } if (!valid_uplo) { std::ostringstream os; os << "KokkosBlas::trsm: uplo = '" << uplo[0] << "'. " << "Valid values include 'U' or 'u' (A is upper triangular), " "'L' or 'l' (A is lower triangular)."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } if (!valid_trans) { std::ostringstream os; @@ -123,7 +124,7 @@ void trsm(const char side[], const char uplo[], const char trans[], << "Valid values include 'N' or 'n' (No transpose), 'T' or 't' " "(Transpose), " "and 'C' or 'c' (Conjugate transpose)."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } if (!valid_diag) { std::ostringstream os; @@ -131,7 +132,7 @@ void trsm(const char side[], const char uplo[], const char trans[], << "Valid values include 'U' or 'u' (the diagonal of A is assumed to be " "unit), " "'N' or 'n' (the diagonal of A is assumed to be non-unit)."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Check compatibility of dimensions at run time. @@ -147,7 +148,7 @@ void trsm(const char side[], const char uplo[], const char trans[], os << "KokkosBlas::trsm: Dimensions of A and B do not match: " << "side: " << side[0] << " A: " << A.extent(0) << " x " << A.extent(1) << " B: " << B.extent(0) << " x " << B.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Return if degenerated matrices are provided diff --git a/src/blas/KokkosBlas_gesv.hpp b/src/blas/KokkosBlas_gesv.hpp index 00b5a1bfa7..e96b965546 100644 --- a/src/blas/KokkosBlas_gesv.hpp +++ b/src/blas/KokkosBlas_gesv.hpp @@ -56,6 +56,7 @@ #include #include "KokkosBlas_gesv_spec.hpp" +#include "KokkosKernels_Error.hpp" namespace KokkosBlas { @@ -110,7 +111,7 @@ void gesv(const AMatrix& A, const BXMV& B, const IPIVV& IPIV) { << "Valid options include zero-extent 1-D view (no pivoting), or 1-D " "View with size of " << A0 << " (partial pivoting)."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Check for no pivoting case. Only MAGMA supports no pivoting interface @@ -122,7 +123,7 @@ void gesv(const AMatrix& A, const BXMV& B, const IPIVV& IPIV) { std::ostringstream os; os << "KokkosBlas::gesv: IPIV: " << IPIV0 << ". " << "BLAS TPL does not support no pivoting."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } #endif #else // not have MAGMA TPL @@ -131,7 +132,7 @@ void gesv(const AMatrix& A, const BXMV& B, const IPIVV& IPIV) { std::ostringstream os; os << "KokkosBlas::gesv: IPIV: " << IPIV0 << ". " << "BLAS TPL does not support no pivoting."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } #endif #endif @@ -142,7 +143,7 @@ void gesv(const AMatrix& A, const BXMV& B, const IPIVV& IPIV) { os << "KokkosBlas::gesv: Dimensions of A, and B do not match: " << " A: " << A.extent(0) << " x " << A.extent(1) << " B: " << B.extent(0) << " x " << B.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } typedef Kokkos::View< diff --git a/src/blas/KokkosBlas_trtri.hpp b/src/blas/KokkosBlas_trtri.hpp index 356aef1655..0402b11104 100644 --- a/src/blas/KokkosBlas_trtri.hpp +++ b/src/blas/KokkosBlas_trtri.hpp @@ -51,6 +51,7 @@ #include "KokkosKernels_helpers.hpp" #include #include +#include "KokkosKernels_Error.hpp" namespace KokkosBlas { @@ -91,7 +92,7 @@ int trtri(const char uplo[], const char diag[], const AViewType& A) { os << "KokkosBlas::trtri: uplo = '" << uplo[0] << "'. " << "Valid values include 'U' or 'u' (A is upper triangular), " "'L' or 'l' (A is lower triangular)."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } if (!valid_diag) { std::ostringstream os; @@ -99,7 +100,7 @@ int trtri(const char uplo[], const char diag[], const AViewType& A) { << "Valid values include 'U' or 'u' (the diagonal of A is assumed to be " "unit), " "'N' or 'n' (the diagonal of A is assumed to be non-unit)."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } int64_t A_m = A.extent(0); @@ -116,7 +117,7 @@ int trtri(const char uplo[], const char diag[], const AViewType& A) { std::ostringstream os; os << "KokkosBlas::trtri: Dimensions of A do not match," << " A: " << A.extent(0) << " x " << A.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Create A matrix view type alias diff --git a/src/common/KokkosKernels_Error.hpp b/src/common/KokkosKernels_Error.hpp new file mode 100644 index 0000000000..061480540a --- /dev/null +++ b/src/common/KokkosKernels_Error.hpp @@ -0,0 +1,78 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Siva Rajamanickam (srajama@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOSKERNELS_ERROR_HPP +#define KOKKOSKERNELS_ERROR_HPP + +#include +#include + +namespace KokkosKernels { +namespace Impl { + +inline void traceback_callstack(std::ostream &msg) { + msg << std::endl << "Traceback functionality not available" << std::endl; +} + +// void traceback_callstack(std::ostream &msg) { +// #ifdef KOKKOS_IMPL_ENABLE_STACKTRACE +// msg << "\nBacktrace:\n"; +// save_stacktrace(); +// print_demangled_saved_stacktrace(msg); +// #else +// msg << "\nTraceback functionality not available\n"; +// #endif +// } + +inline void throw_runtime_exception(const std::string &msg) { + std::ostringstream o; + o << msg; + traceback_callstack(o); + throw std::runtime_error(o.str()); +} + +} // namespace Impl +} // namespace KokkosKernels + +#endif // KOKKOSKERNELS_ERROR_HPP diff --git a/src/graph/KokkosGraph_Distance2ColorHandle.hpp b/src/graph/KokkosGraph_Distance2ColorHandle.hpp index 112717085a..757a9e0dfe 100644 --- a/src/graph/KokkosGraph_Distance2ColorHandle.hpp +++ b/src/graph/KokkosGraph_Distance2ColorHandle.hpp @@ -48,6 +48,7 @@ #include #include #include +#include #ifndef _GRAPHCOLORDISTANCE2HANDLE_HPP #define _GRAPHCOLORDISTANCE2HANDLE_HPP @@ -167,7 +168,7 @@ class GraphColorDistance2Handle { std::string message = "Distance-2 Graph Coloring Handle does not currently support " "different mem spaces"; - Kokkos::Impl::throw_runtime_exception(message); + KokkosKernels::Impl::throw_runtime_exception(message); } } diff --git a/src/sparse/KokkosSparse_BsrMatrix.hpp b/src/sparse/KokkosSparse_BsrMatrix.hpp index 3bf1f2db1a..a87071867b 100644 --- a/src/sparse/KokkosSparse_BsrMatrix.hpp +++ b/src/sparse/KokkosSparse_BsrMatrix.hpp @@ -61,6 +61,7 @@ #include "Kokkos_StaticCrsGraph.hpp" #include "Kokkos_ArithTraits.hpp" #include "KokkosSparse_CrsMatrix.hpp" +#include "KokkosKernels_Error.hpp" namespace KokkosSparse { @@ -475,7 +476,7 @@ class BsrMatrix { if (blockDim_ < 1) { std::ostringstream os; os << "KokkosSparse::BsrMatrix: Inappropriate block size: " << blockDim_; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } @@ -513,7 +514,7 @@ class BsrMatrix { if (blockDim_ < 1) { std::ostringstream os; os << "KokkosSparse::BsrMatrix: Inappropriate block size: " << blockDim_; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } if ((ncols % blockDim_ != 0) || (nrows % blockDim_ != 0)) { @@ -637,7 +638,7 @@ class BsrMatrix { if (blockDim_ < 1) { std::ostringstream os; os << "KokkosSparse::BsrMatrix: Inappropriate block size: " << blockDim_; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } const ordinal_type actualNumRows = @@ -680,7 +681,7 @@ class BsrMatrix { if (blockDim_ < 1) { std::ostringstream os; os << "KokkosSparse::BsrMatrix: Inappropriate block size: " << blockDim_; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } @@ -701,7 +702,7 @@ class BsrMatrix { if (blockDim_ < 1) { std::ostringstream os; os << "KokkosSparse::BsrMatrix: Inappropriate block size: " << blockDim_; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } assert( diff --git a/src/sparse/KokkosSparse_gauss_seidel.hpp b/src/sparse/KokkosSparse_gauss_seidel.hpp index 5cc176d74b..4a52092db2 100644 --- a/src/sparse/KokkosSparse_gauss_seidel.hpp +++ b/src/sparse/KokkosSparse_gauss_seidel.hpp @@ -47,6 +47,7 @@ #include "KokkosSparse_gauss_seidel_spec.hpp" #include "KokkosKernels_Handle.hpp" #include "KokkosKernels_helpers.hpp" +#include "KokkosKernels_Error.hpp" namespace KokkosSparse { @@ -357,7 +358,7 @@ void symmetric_gauss_seidel_apply( os << "KokkosSparse::symmetric_gauss_seidel_apply: " << "X has " << x_lhs_output_vec.extent(1) << "columns, Y has " << y_rhs_input_vec.extent(1) << " columns."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } typedef typename KernelHandle::const_size_type c_size_t; @@ -452,7 +453,7 @@ void symmetric_block_gauss_seidel_apply( "and Y do not match: " << "X has " << x_lhs_output_vec.extent(1) << "columns, Y has " << y_rhs_input_vec.extent(1) << " columns."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } auto gsHandle = handle->get_point_gs_handle(); if (gsHandle->get_algorithm_type() == GS_CLUSTER) { @@ -523,7 +524,7 @@ void forward_sweep_gauss_seidel_apply( "Y do not match: " << "X has " << x_lhs_output_vec.extent(1) << "columns, Y has " << y_rhs_input_vec.extent(1) << " columns."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } typedef typename KernelHandle::const_size_type c_size_t; @@ -619,7 +620,7 @@ void forward_sweep_block_gauss_seidel_apply( "X and Y do not match: " << "X has " << x_lhs_output_vec.extent(1) << "columns, Y has " << y_rhs_input_vec.extent(1) << " columns."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } auto gsHandle = handle->get_point_gs_handle(); @@ -690,7 +691,7 @@ void backward_sweep_gauss_seidel_apply( "and Y do not match: " << "X has " << x_lhs_output_vec.extent(1) << "columns, Y has " << y_rhs_input_vec.extent(1) << " columns."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } typedef typename KernelHandle::const_size_type c_size_t; @@ -786,7 +787,7 @@ void backward_sweep_block_gauss_seidel_apply( "of X and Y do not match: " << "X has " << x_lhs_output_vec.extent(1) << "columns, Y has " << y_rhs_input_vec.extent(1) << " columns."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } auto gsHandle = handle->get_point_gs_handle(); if (gsHandle->get_algorithm_type() == GS_CLUSTER) { diff --git a/src/sparse/KokkosSparse_spiluk.hpp b/src/sparse/KokkosSparse_spiluk.hpp index 9abecce192..f1ed7b3e0f 100644 --- a/src/sparse/KokkosSparse_spiluk.hpp +++ b/src/sparse/KokkosSparse_spiluk.hpp @@ -56,6 +56,7 @@ //#include "KokkosSparse_spiluk_handle.hpp" #include "KokkosKernels_helpers.hpp" +#include "KokkosKernels_Error.hpp" #include "KokkosSparse_spiluk_symbolic_spec.hpp" #include "KokkosSparse_spiluk_numeric_spec.hpp" @@ -194,7 +195,7 @@ void spiluk_symbolic(KernelHandle* handle, std::ostringstream os; os << "KokkosSparse::Experimental::spiluk_symbolic: fill_lev: " << fill_lev << ". Valid value is >= 0."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } typedef typename KernelHandle::const_size_type c_size_t; @@ -448,7 +449,7 @@ void spiluk_numeric(KernelHandle* handle, std::ostringstream os; os << "KokkosSparse::Experimental::spiluk_numeric: fill_lev: " << fill_lev << ". Valid value is >= 0."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // Check if symbolic has been called @@ -456,7 +457,7 @@ void spiluk_numeric(KernelHandle* handle, std::ostringstream os; os << "KokkosSparse::Experimental::spiluk_numeric: spiluk_symbolic must be " "called before spiluk_numeric."; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } typedef typename KernelHandle::const_size_type c_size_t; diff --git a/src/sparse/KokkosSparse_spmv.hpp b/src/sparse/KokkosSparse_spmv.hpp index 1862be1d20..e29ecc9808 100644 --- a/src/sparse/KokkosSparse_spmv.hpp +++ b/src/sparse/KokkosSparse_spmv.hpp @@ -58,6 +58,7 @@ #include "KokkosBlas1_scal.hpp" #include "KokkosKernels_Utils.hpp" #include "KokkosKernels_MemSpaceUtils.hpp" +#include "KokkosKernels_Error.hpp" namespace KokkosSparse { @@ -96,7 +97,7 @@ void spmv(KokkosKernels::Experimental::Controls controls, const char mode[], << ", A: " << A.numRows() << " x " << A.numCols() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else { if ((x.extent(1) != y.extent(1)) || @@ -107,7 +108,7 @@ void spmv(KokkosKernels::Experimental::Controls controls, const char mode[], << ", A: " << A.numRows() << " x " << A.numCols() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } @@ -272,7 +273,7 @@ void spmv(KokkosKernels::Experimental::Controls controls, const char mode[], << A.numCols() * A.blockDim() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else { if ((x.extent(1) != y.extent(1)) || @@ -287,7 +288,7 @@ void spmv(KokkosKernels::Experimental::Controls controls, const char mode[], << A.numCols() * A.blockDim() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } // @@ -383,7 +384,7 @@ void spmv(KokkosKernels::Experimental::Controls controls, const char mode[], << A.numCols() * A.blockDim() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else { if ((x.extent(1) != y.extent(1)) || @@ -398,7 +399,7 @@ void spmv(KokkosKernels::Experimental::Controls controls, const char mode[], << A.numCols() * A.blockDim() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } // @@ -632,7 +633,7 @@ void spmv(KokkosKernels::Experimental::Controls /*controls*/, const char mode[], << ", A: " << A.numRows() << " x " << A.numCols() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else { if ((x.extent(1) != y.extent(1)) || @@ -643,7 +644,7 @@ void spmv(KokkosKernels::Experimental::Controls /*controls*/, const char mode[], << ", A: " << A.numRows() << " x " << A.numCols() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } @@ -757,7 +758,7 @@ void spmv(KokkosKernels::Experimental::Controls controls, const char mode[], << A.numCols() * A.blockDim() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else { if ((x.extent(1) != y.extent(1)) || @@ -772,7 +773,7 @@ void spmv(KokkosKernels::Experimental::Controls controls, const char mode[], << A.numCols() * A.blockDim() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } // @@ -943,7 +944,7 @@ void spmv(KokkosKernels::Experimental::Controls controls, const char mode[], << A.numCols() * A.blockDim() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else { if ((x.extent(1) != y.extent(1)) || @@ -958,7 +959,7 @@ void spmv(KokkosKernels::Experimental::Controls controls, const char mode[], << A.numCols() * A.blockDim() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } // @@ -1166,7 +1167,7 @@ void spmv_struct(const char mode[], const int stencil_type, << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else { if ((x.extent(1) != y.extent(1)) || @@ -1178,7 +1179,7 @@ void spmv_struct(const char mode[], const int stencil_type, << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } @@ -1356,7 +1357,7 @@ void spmv_struct(const char mode[], const int stencil_type, << ", A: " << A.numRows() << " x " << A.numCols() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else { if ((x.extent(1) != y.extent(1)) || @@ -1367,7 +1368,7 @@ void spmv_struct(const char mode[], const int stencil_type, << ", A: " << A.numRows() << " x " << A.numCols() << ", x: " << x.extent(0) << " x " << x.extent(1) << ", y: " << y.extent(0) << " x " << y.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } diff --git a/src/sparse/KokkosSparse_trsv.hpp b/src/sparse/KokkosSparse_trsv.hpp index 92f6b263f3..eda3501ad0 100644 --- a/src/sparse/KokkosSparse_trsv.hpp +++ b/src/sparse/KokkosSparse_trsv.hpp @@ -55,6 +55,7 @@ #include #include "KokkosSparse_trsv_spec.hpp" +#include "KokkosKernels_Error.hpp" namespace KokkosSparse { @@ -106,20 +107,20 @@ void trsv(const char uplo[], const char trans[], const char diag[], uplo[0] != 'l') { std::ostringstream os; os << "Invalid uplo[0] = \'" << uplo << "\'"; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } if (zero != numRows && trans[0] != 'C' && trans[0] != 'c' && trans[0] != 'T' && trans[0] != 't' && trans[0] != 'N' && trans[0] != 'n') { std::ostringstream os; os << "Invalid trans[0] = \'" << trans << "\'"; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } if (zero != numRows && diag[0] != 'U' && diag[0] != 'u' && diag[0] != 'N' && diag[0] != 'n') { std::ostringstream os; os << "Invalid diag[0] = \'" << diag << "\'"; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } /* typedef typename BMV::size_type size_type; @@ -133,7 +134,7 @@ void trsv(const char uplo[], const char trans[], const char diag[], << "A is " << numRows << " x " << numCols << ", x is " << x.extent(0) << " x " << x.extent(1) << ", and b is " << b.extent(0) << " x " << b.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } if (transpose && (numRows != x.extent(0) || numCols != b.extent(0))) { std::ostringstream os; @@ -141,7 +142,7 @@ void trsv(const char uplo[], const char trans[], const char diag[], << "A is " << numRows << " x " << numCols << ", x is " << x.extent(0) << " x " << x.extent(1) << ", and b is " << b.extent(0) << " x " << b.extent(1); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } typedef KokkosSparse::CrsMatrix< diff --git a/src/sparse/impl/KokkosSparse_spiluk_symbolic_impl.hpp b/src/sparse/impl/KokkosSparse_spiluk_symbolic_impl.hpp index 2d4588e646..ff464951c7 100644 --- a/src/sparse/impl/KokkosSparse_spiluk_symbolic_impl.hpp +++ b/src/sparse/impl/KokkosSparse_spiluk_symbolic_impl.hpp @@ -52,6 +52,7 @@ #include #include #include +#include //#define SYMBOLIC_OUTPUT_INFO @@ -308,7 +309,7 @@ void iluk_symbolic(IlukHandle& thandle, os << "KokkosSparse::Experimental::spiluk_symbolic: U_entries's extent " "must be larger than " << U_entries_d.extent(0); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } // U diag entry U_entries(cntU) = i; @@ -334,7 +335,7 @@ void iluk_symbolic(IlukHandle& thandle, os << "KokkosSparse::Experimental::spiluk_symbolic: L_entries's extent " "must be larger than " << L_entries_d.extent(0); - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } for (size_type k = 0; k < lenl; ++k) { L_entries(cntL) = h_iL(k); diff --git a/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp b/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp index ec54f3bb6b..ceda2db759 100644 --- a/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp +++ b/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp @@ -45,6 +45,8 @@ #ifndef KOKKOSSPARSE_IMPL_SPMV_BSRMATRIX_IMPL_HPP_ #define KOKKOSSPARSE_IMPL_SPMV_BSRMATRIX_IMPL_HPP_ +#include "KokkosKernels_Error.hpp" + #if defined(KOKKOS_ENABLE_CUDA) && \ (defined(KOKKOS_ARCH_VOLTA) || defined(KOKKOS_ARCH_AMPERE)) @@ -500,7 +502,8 @@ struct BsrMatrixSpMVTensorCoreDispatcher { // to be used to avoid instantiating on unsupported types static void tag_dispatch(std::false_type, YScalar, AMatrix, XMatrix, YScalar, YMatrix) { - Kokkos::Impl::throw_runtime_exception("unsupported for complex types"); + KokkosKernels::Impl::throw_runtime_exception( + "unsupported for complex types"); } /*true if T1, T2, or T3 are complex*/ diff --git a/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp b/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp index cbbbd39f12..30edf4131e 100644 --- a/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp +++ b/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp @@ -50,6 +50,7 @@ #include "KokkosSparse_BsrMatrix.hpp" #include "KokkosKernels_Controls.hpp" +#include "KokkosKernels_Error.hpp" #if !defined(KOKKOSKERNELS_ETI_ONLY) || KOKKOSKERNELS_IMPL_COMPILE_LIBRARY #include #endif @@ -272,7 +273,7 @@ struct SPMV_MV_BSRMATRIX(alpha, A, x, beta, y); } else { - Kokkos::Impl::throw_runtime_exception( + KokkosKernels::Impl::throw_runtime_exception( "Invalid Transpose Mode for KokkosSparse::spmv()"); } } @@ -1517,7 +1518,7 @@ static void spmv_alpha_beta_mv( spmv_alpha_beta_mv_transpose(alpha, A, x, beta, y); } else { - Kokkos::Impl::throw_runtime_exception( + KokkosKernels::Impl::throw_runtime_exception( "Invalid Transpose Mode for KokkosSparse::spmv()"); } } diff --git a/src/sparse/impl/KokkosSparse_spmv_struct_impl.hpp b/src/sparse/impl/KokkosSparse_spmv_struct_impl.hpp index 53c6da72df..bbb89ad1f9 100644 --- a/src/sparse/impl/KokkosSparse_spmv_struct_impl.hpp +++ b/src/sparse/impl/KokkosSparse_spmv_struct_impl.hpp @@ -49,6 +49,7 @@ #include "KokkosKernels_ExecSpaceUtils.hpp" #include "KokkosBlas1_scal.hpp" #include "KokkosSparse_CrsMatrix.hpp" +#include "KokkosKernels_Error.hpp" namespace KokkosSparse { namespace Impl { @@ -941,7 +942,7 @@ static void spmv_struct_beta( spmv_struct_beta_transpose( stencil_type, structure, alpha, A, x, beta, y); } else { - Kokkos::Impl::throw_runtime_exception( + KokkosKernels::Impl::throw_runtime_exception( "Invalid Transpose Mode for KokkosSparse::spmv_struct()"); } } @@ -1475,7 +1476,7 @@ static void spmv_alpha_beta_mv_struct( spmv_alpha_beta_mv_struct_transpose(alpha, A, x, beta, y); } else { - Kokkos::Impl::throw_runtime_exception( + KokkosKernels::Impl::throw_runtime_exception( "Invalid Transpose Mode for KokkosSparse::spmv()"); } } diff --git a/src/stage/blas3/Kokkos_Blas3.hpp b/src/stage/blas3/Kokkos_Blas3.hpp index e1a423c1cd..cf9dd9a1bf 100644 --- a/src/stage/blas3/Kokkos_Blas3.hpp +++ b/src/stage/blas3/Kokkos_Blas3.hpp @@ -48,6 +48,7 @@ #include #include // requires C++11 +#include "KokkosKernels_Error.hpp" namespace KokkosBlas { @@ -79,7 +80,7 @@ void gemm(const char transA, const char transB, AMat::const_value_type alpha, << "(" << b.extent(0) << "," << b.extent(1) << ")" << ", C: " << "(" << c.extent(0) << "," << c.extent(1) << ")"; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else if (rank == 3) { if (a.extent(1) != c.extent(1) || a.extent(2) != b.extent(1) || @@ -97,7 +98,7 @@ void gemm(const char transA, const char transB, AMat::const_value_type alpha, << ", C: " << "(" << c.extent(0) << "," << c.extent(1) << "," << c.extent(2) << ")"; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } @@ -115,7 +116,7 @@ void gemm(const char transA, const char transB, AMat::const_value_type alpha, << "(" << b.extent(0) << "," << b.extent(1) << ")" << ", C: " << "(" << c.extent(0) << "," << c.extent(1) << ")"; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else if (rank == 3) { if (a.extent(1) != c.extent(1) || a.extent(2) != b.extent(2) || @@ -133,7 +134,7 @@ void gemm(const char transA, const char transB, AMat::const_value_type alpha, << ", C: " << "(" << c.extent(0) << "," << c.extent(1) << "," << c.extent(2) << ")"; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } @@ -151,7 +152,7 @@ void gemm(const char transA, const char transB, AMat::const_value_type alpha, << "(" << b.extent(0) << "," << b.extent(1) << ")" << ", C: " << "(" << c.extent(0) << "," << c.extent(1) << ")"; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else if (rank == 3) { if (a.extent(2) != c.extent(1) || a.extent(1) != b.extent(1) || @@ -169,7 +170,7 @@ void gemm(const char transA, const char transB, AMat::const_value_type alpha, << ", C: " << "(" << c.extent(0) << "," << c.extent(1) << "," << c.extent(2) << ")"; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } @@ -187,7 +188,7 @@ void gemm(const char transA, const char transB, AMat::const_value_type alpha, << "(" << b.extent(0) << "," << b.extent(1) << ")" << ", C: " << "(" << c.extent(0) << "," << c.extent(1) << ")"; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } else if (rank == 3) { if (a.extent(2) != c.extent(1) || a.extent(1) != b.extent(2) || @@ -205,7 +206,7 @@ void gemm(const char transA, const char transB, AMat::const_value_type alpha, << ", C: " << "(" << c.extent(0) << "," << c.extent(1) << "," << c.extent(2) << ")"; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } } @@ -214,7 +215,7 @@ void gemm(const char transA, const char transB, AMat::const_value_type alpha, os << "KokkosBlas::gemm: values for transA or transB should be T t or N n " "you have input: " << "TransA:" << transA << " TransB:" << transB; - Kokkos::Impl::throw_runtime_exception(os.str()); + KokkosKernels::Impl::throw_runtime_exception(os.str()); } typedef Kokkos::View - struct fill_1D_matrix_functor { - - // Define types used by the CrsMatrix - typedef typename CrsMatrix_t::execution_space execution_space; - typedef typename CrsMatrix_t::row_map_type::non_const_type row_map_view_t; - typedef typename CrsMatrix_t::index_type::non_const_type cols_view_t; - typedef typename CrsMatrix_t::values_type::non_const_type scalar_view_t; - typedef typename CrsMatrix_t::non_const_size_type size_type; - typedef typename CrsMatrix_t::non_const_ordinal_type ordinal_type; - - // Dispatch tags - struct interiorTag{}; - struct exteriorTag{}; - - // Internal variables and temporaries - const ordinal_type numNodes; - const int leftBC, rightBC; - const ordinal_type interiorStencilLength, cornerStencilLength; - ordinal_type numInterior; - size_type numEntries; - - // Matrix views - row_map_view_t rowmap; - cols_view_t columns; - scalar_view_t values; - - fill_1D_matrix_functor(const ordinal_type numNodes_, - const int leftBC_, const int rightBC_, - const row_map_view_t rowmap_, const cols_view_t columns_, - const scalar_view_t values_) : - numNodes(numNodes_), leftBC(leftBC_), rightBC(rightBC_), - interiorStencilLength(3), cornerStencilLength(2), - rowmap(rowmap_), columns(columns_), values(values_) { - - if(numNodes == 1) { - std::ostringstream os; - os << "You need at least two points per direction to obtain a valid discretization !" - << std::endl; - throw std::runtime_error(os.str()); - } - - numInterior = numNodes - 2; - numEntries = numInterior*interiorStencilLength + 2*cornerStencilLength; - +enum { FD, FE }; + +template +struct fill_1D_matrix_functor { + // Define types used by the CrsMatrix + typedef typename CrsMatrix_t::execution_space execution_space; + typedef typename CrsMatrix_t::row_map_type::non_const_type row_map_view_t; + typedef typename CrsMatrix_t::index_type::non_const_type cols_view_t; + typedef typename CrsMatrix_t::values_type::non_const_type scalar_view_t; + typedef typename CrsMatrix_t::non_const_size_type size_type; + typedef typename CrsMatrix_t::non_const_ordinal_type ordinal_type; + + // Dispatch tags + struct interiorTag {}; + struct exteriorTag {}; + + // Internal variables and temporaries + const ordinal_type numNodes; + const int leftBC, rightBC; + const ordinal_type interiorStencilLength, cornerStencilLength; + ordinal_type numInterior; + size_type numEntries; + + // Matrix views + row_map_view_t rowmap; + cols_view_t columns; + scalar_view_t values; + + fill_1D_matrix_functor(const ordinal_type numNodes_, const int leftBC_, + const int rightBC_, const row_map_view_t rowmap_, + const cols_view_t columns_, + const scalar_view_t values_) + : numNodes(numNodes_), + leftBC(leftBC_), + rightBC(rightBC_), + interiorStencilLength(3), + cornerStencilLength(2), + rowmap(rowmap_), + columns(columns_), + values(values_) { + if (numNodes == 1) { + std::ostringstream os; + os << "You need at least two points per direction to obtain a valid " + "discretization !" + << std::endl; + throw std::runtime_error(os.str()); } - void compute() { - // Fill interior points - if(0 < numInterior) { - Kokkos::RangePolicy interiorPolicy(0, numInterior); - Kokkos::parallel_for("Fill 1D matrix: interior points", interiorPolicy, *this); - } - - // Fill exterior points a.k.a. boundary points - Kokkos::RangePolicy exteriorPolicy(0, 1); - Kokkos::parallel_for("Fill 1D matrix: exterior points", exteriorPolicy, *this); + numInterior = numNodes - 2; + numEntries = numInterior * interiorStencilLength + 2 * cornerStencilLength; + } + + void compute() { + // Fill interior points + if (0 < numInterior) { + Kokkos::RangePolicy interiorPolicy( + 0, numInterior); + Kokkos::parallel_for("Fill 1D matrix: interior points", interiorPolicy, + *this); } - KOKKOS_INLINE_FUNCTION - void operator() (const interiorTag&, const size_type idx) const { - const ordinal_type rowIdx = idx + 1; // Offset by one since first node has BC - const size_type rowOffset = rowIdx*interiorStencilLength + cornerStencilLength; - - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 3) = rowIdx - 1; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + 1; - - // Fill values - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 2.0; - values(rowOffset - 1) = -1.0; + // Fill exterior points a.k.a. boundary points + Kokkos::RangePolicy exteriorPolicy(0, 1); + Kokkos::parallel_for("Fill 1D matrix: exterior points", exteriorPolicy, + *this); + } + + KOKKOS_INLINE_FUNCTION + void operator()(const interiorTag&, const size_type idx) const { + const ordinal_type rowIdx = + idx + 1; // Offset by one since first node has BC + const size_type rowOffset = + rowIdx * interiorStencilLength + cornerStencilLength; + + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 3) = rowIdx - 1; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + 1; + + // Fill values + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 2.0; + values(rowOffset - 1) = -1.0; + } + + KOKKOS_INLINE_FUNCTION + void operator()(const exteriorTag&, const size_type /*idx*/) const { + // LeftBC + rowmap(1) = 2; + + columns(0) = 0; + columns(1) = 1; + if (leftBC == 1) { + values(0) = 1.0; + values(1) = 0.0; + } else { + values(0) = 1.0; + values(1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const exteriorTag&, const size_type /*idx*/) const { - // LeftBC - rowmap(1) = 2; - - columns(0) = 0; - columns(1) = 1; - if(leftBC == 1) { - values(0) = 1.0; - values(1) = 0.0; - } else { - values(0) = 1.0; - values(1) = -1.0; - } + // RightBC + rowmap(numNodes) = numEntries; - // RightBC - rowmap(numNodes) = numEntries; - - columns(numEntries - 2) = numNodes - 2; - columns(numEntries - 1) = numNodes - 1; - if(rightBC == 1) { - values(numEntries - 2) = 0.0; - values(numEntries - 1) = 1.0; - } else { - values(numEntries - 2) = -1.0; - values(numEntries - 1) = 1.0; - } + columns(numEntries - 2) = numNodes - 2; + columns(numEntries - 1) = numNodes - 1; + if (rightBC == 1) { + values(numEntries - 2) = 0.0; + values(numEntries - 1) = 1.0; + } else { + values(numEntries - 2) = -1.0; + values(numEntries - 1) = 1.0; + } + } +}; + +template +CrsMatrix_t generate_structured_matrix1D(const mat_structure& structure) { + typedef typename CrsMatrix_t::StaticCrsGraphType graph_t; + typedef typename CrsMatrix_t::row_map_type::non_const_type row_map_view_t; + typedef typename CrsMatrix_t::index_type::non_const_type cols_view_t; + typedef typename CrsMatrix_t::values_type::non_const_type scalar_view_t; + typedef typename CrsMatrix_t::non_const_size_type size_type; + typedef typename CrsMatrix_t::non_const_ordinal_type ordinal_type; + + // Extract geometric data + const ordinal_type nx = structure(0, 0); + const ordinal_type numNodes = nx; + const ordinal_type leftBC = structure(0, 1); + const ordinal_type rightBC = structure(0, 2); + const ordinal_type numInterior = (nx - leftBC - rightBC); + const ordinal_type numCorner = leftBC + rightBC; + const ordinal_type interiorStencilLength = 3, cornerStencilLength = 2; + const size_type numEntries = + numInterior * interiorStencilLength + numCorner * cornerStencilLength; + + // Create matrix data + row_map_view_t rowmap_view("rowmap_view", numNodes + 1); + cols_view_t columns_view("colsmap_view", numEntries); + scalar_view_t values_view("values_view", numEntries); + + fill_1D_matrix_functor fill_matrix( + numNodes, leftBC, rightBC, rowmap_view, columns_view, values_view); + fill_matrix.compute(); + + graph_t static_graph(columns_view, rowmap_view); + std::string name = "CrsMatrixFE"; + + return CrsMatrix_t(name, numNodes, values_view, static_graph); + +} // generate_structured_matrix1D + +template +struct fill_2D_matrix_functor { + // Define types used by the CrsMatrix + using execution_space = typename CrsMatrix_t::execution_space; + using row_map_view_t = typename CrsMatrix_t::row_map_type::non_const_type; + using cols_view_t = typename CrsMatrix_t::index_type::non_const_type; + using scalar_view_t = typename CrsMatrix_t::values_type::non_const_type; + using size_type = typename CrsMatrix_t::non_const_size_type; + using ordinal_type = typename CrsMatrix_t::non_const_ordinal_type; + + // Finite difference dispatch tags + struct interiorFDTag {}; + + struct xEdgeFDTag {}; + struct yEdgeFDTag {}; + + struct cornerFDTag {}; + + // Finite element dispatch tags + struct interiorFETag {}; + + struct xEdgeFETag {}; + struct yEdgeFETag {}; + + struct cornerFETag {}; + + // Internal variables and temporaries + const int stencil_type; + const ordinal_type nx, ny; + const int leftBC, rightBC, bottomBC, topBC; + + // Matrix views + row_map_view_t rowmap; + cols_view_t columns; + scalar_view_t values; + + ordinal_type interiorStencilLength, edgeStencilLength, cornerStencilLength; + ordinal_type numInterior; + ordinal_type numXEdge; + ordinal_type numYEdge; + ordinal_type numCorner; + ordinal_type numEntriesPerGridRow; + ordinal_type numEntriesBottomRow; + size_type numEntries; + + fill_2D_matrix_functor(const int stencil_type_, const ordinal_type nx_, + const ordinal_type ny_, const int leftBC_, + const int rightBC_, const int bottomBC_, + const int topBC_, const row_map_view_t rowmap_, + const cols_view_t columns_, + const scalar_view_t values_) + : stencil_type(stencil_type_), + nx(nx_), + ny(ny_), + leftBC(leftBC_), + rightBC(rightBC_), + bottomBC(bottomBC_), + topBC(topBC_), + rowmap(rowmap_), + columns(columns_), + values(values_) { + if (nx == 1 || ny == 1) { + std::ostringstream os; + os << "You need at least two points per direction to obtain a valid " + "discretization!" + << std::endl; + throw std::runtime_error(os.str()); } - }; - - template - CrsMatrix_t generate_structured_matrix1D(const mat_structure& structure) { - - typedef typename CrsMatrix_t::StaticCrsGraphType graph_t; - typedef typename CrsMatrix_t::row_map_type::non_const_type row_map_view_t; - typedef typename CrsMatrix_t::index_type::non_const_type cols_view_t; - typedef typename CrsMatrix_t::values_type::non_const_type scalar_view_t; - typedef typename CrsMatrix_t::non_const_size_type size_type; - typedef typename CrsMatrix_t::non_const_ordinal_type ordinal_type; - - // Extract geometric data - const ordinal_type nx = structure(0,0); - const ordinal_type numNodes = nx; - const ordinal_type leftBC = structure(0,1); - const ordinal_type rightBC = structure(0,2); - const ordinal_type numInterior = (nx - leftBC - rightBC); - const ordinal_type numCorner = leftBC + rightBC; - const ordinal_type interiorStencilLength = 3, cornerStencilLength = 2; - const size_type numEntries = numInterior*interiorStencilLength + numCorner*cornerStencilLength; - - // Create matrix data - row_map_view_t rowmap_view ("rowmap_view", numNodes + 1); - cols_view_t columns_view("colsmap_view", numEntries); - scalar_view_t values_view ("values_view", numEntries); - - fill_1D_matrix_functor fill_matrix(numNodes, leftBC, rightBC, - rowmap_view, columns_view, values_view); - fill_matrix.compute(); - - graph_t static_graph (columns_view, rowmap_view); - std::string name = "CrsMatrixFE"; - - return CrsMatrix_t(name, numNodes, values_view, static_graph); - - } // generate_structured_matrix1D - - template - struct fill_2D_matrix_functor { - - // Define types used by the CrsMatrix - using execution_space = typename CrsMatrix_t::execution_space; - using row_map_view_t = typename CrsMatrix_t::row_map_type::non_const_type; - using cols_view_t = typename CrsMatrix_t::index_type::non_const_type; - using scalar_view_t = typename CrsMatrix_t::values_type::non_const_type; - using size_type = typename CrsMatrix_t::non_const_size_type; - using ordinal_type = typename CrsMatrix_t::non_const_ordinal_type; - - // Finite difference dispatch tags - struct interiorFDTag{}; - - struct xEdgeFDTag{}; - struct yEdgeFDTag{}; - - struct cornerFDTag{}; - - // Finite element dispatch tags - struct interiorFETag{}; - - struct xEdgeFETag{}; - struct yEdgeFETag{}; - - struct cornerFETag{}; - - // Internal variables and temporaries - const int stencil_type; - const ordinal_type nx, ny; - const int leftBC, rightBC, bottomBC, topBC; - - // Matrix views - row_map_view_t rowmap; - cols_view_t columns; - scalar_view_t values; - - ordinal_type interiorStencilLength, edgeStencilLength, cornerStencilLength; - ordinal_type numInterior; - ordinal_type numXEdge; - ordinal_type numYEdge; - ordinal_type numCorner; - ordinal_type numEntriesPerGridRow; - ordinal_type numEntriesBottomRow; - size_type numEntries; - - fill_2D_matrix_functor(const int stencil_type_, const ordinal_type nx_, - const ordinal_type ny_, - const int leftBC_, const int rightBC_, - const int bottomBC_, const int topBC_, - const row_map_view_t rowmap_, const cols_view_t columns_, - const scalar_view_t values_) : - stencil_type(stencil_type_), nx(nx_), ny(ny_), - leftBC(leftBC_), rightBC(rightBC_), bottomBC(bottomBC_), topBC(topBC_), - rowmap(rowmap_), columns(columns_), values(values_) { - - if(nx == 1 || ny == 1) { - std::ostringstream os; - os << "You need at least two points per direction to obtain a valid discretization!" - << std::endl; - throw std::runtime_error(os.str()); - } - - if(stencil_type == FD) { - interiorStencilLength = 5; - edgeStencilLength = 4; - cornerStencilLength = 3; - } else if(stencil_type == FE) { - interiorStencilLength = 9; - edgeStencilLength = 6; - cornerStencilLength = 4; - } - - numInterior = (nx - 2)*(ny - 2); - numXEdge = nx - 2; - numYEdge = ny - 2; - numCorner = 4; - - numEntriesPerGridRow = (nx - 2)*interiorStencilLength - + 2*edgeStencilLength; - - numEntriesBottomRow = (nx - 2)*edgeStencilLength - + 2*cornerStencilLength; - numEntries = numInterior*interiorStencilLength - + (2*numXEdge + 2*numYEdge)*edgeStencilLength - + numCorner*cornerStencilLength; + if (stencil_type == FD) { + interiorStencilLength = 5; + edgeStencilLength = 4; + cornerStencilLength = 3; + } else if (stencil_type == FE) { + interiorStencilLength = 9; + edgeStencilLength = 6; + cornerStencilLength = 4; } - void compute() { - // Fill interior points - if(0 < numInterior) { - if(stencil_type == FD) { - Kokkos::RangePolicy policy(0, numInterior); - Kokkos::parallel_for("Fill 2D FD matrix: interior points", policy, *this); - } else if(stencil_type == FE) { - Kokkos::RangePolicy policy(0, numInterior); - Kokkos::parallel_for("Fill 2D FE matrix: interior points", policy, *this); - } + numInterior = (nx - 2) * (ny - 2); + numXEdge = nx - 2; + numYEdge = ny - 2; + numCorner = 4; + + numEntriesPerGridRow = + (nx - 2) * interiorStencilLength + 2 * edgeStencilLength; + + numEntriesBottomRow = + (nx - 2) * edgeStencilLength + 2 * cornerStencilLength; + + numEntries = numInterior * interiorStencilLength + + (2 * numXEdge + 2 * numYEdge) * edgeStencilLength + + numCorner * cornerStencilLength; + } + + void compute() { + // Fill interior points + if (0 < numInterior) { + if (stencil_type == FD) { + Kokkos::RangePolicy policy(0, + numInterior); + Kokkos::parallel_for("Fill 2D FD matrix: interior points", policy, + *this); + } else if (stencil_type == FE) { + Kokkos::RangePolicy policy(0, + numInterior); + Kokkos::parallel_for("Fill 2D FE matrix: interior points", policy, + *this); } + } - // Fill x-edge points - if(0 < numXEdge) { - if(stencil_type == FD) { - Kokkos::RangePolicy policy(0, numXEdge); - Kokkos::parallel_for("Fill 2D FD matrix: x-edge points", policy, *this); - } else if(stencil_type == FE) { - Kokkos::RangePolicy policy(0, numXEdge); - Kokkos::parallel_for("Fill 2D FE matrix: x-edge points", policy, *this); - } + // Fill x-edge points + if (0 < numXEdge) { + if (stencil_type == FD) { + Kokkos::RangePolicy policy(0, numXEdge); + Kokkos::parallel_for("Fill 2D FD matrix: x-edge points", policy, *this); + } else if (stencil_type == FE) { + Kokkos::RangePolicy policy(0, numXEdge); + Kokkos::parallel_for("Fill 2D FE matrix: x-edge points", policy, *this); } + } - // Fill y-edge points - if(0 < numYEdge) { - if(stencil_type == FD) { - Kokkos::RangePolicy policy(0, numYEdge); - Kokkos::parallel_for("Fill 2D FD matrix: y-edge points", policy, *this); - } else if(stencil_type == FE) { - Kokkos::RangePolicy policy(0, numYEdge); - Kokkos::parallel_for("Fill 2D FE matrix: y-edge points", policy, *this); - } + // Fill y-edge points + if (0 < numYEdge) { + if (stencil_type == FD) { + Kokkos::RangePolicy policy(0, numYEdge); + Kokkos::parallel_for("Fill 2D FD matrix: y-edge points", policy, *this); + } else if (stencil_type == FE) { + Kokkos::RangePolicy policy(0, numYEdge); + Kokkos::parallel_for("Fill 2D FE matrix: y-edge points", policy, *this); } + } - // Fill corner points - if(0 < numCorner) { - if(stencil_type == FD) { - Kokkos::RangePolicy policy(0, 1); - Kokkos::parallel_for("Fill 2D FD matrix: corner points", policy, *this); - } else if(stencil_type == FE) { - Kokkos::RangePolicy policy(0, 1); - Kokkos::parallel_for("Fill 2D FE matrix: corner points", policy, *this); - } + // Fill corner points + if (0 < numCorner) { + if (stencil_type == FD) { + Kokkos::RangePolicy policy(0, 1); + Kokkos::parallel_for("Fill 2D FD matrix: corner points", policy, *this); + } else if (stencil_type == FE) { + Kokkos::RangePolicy policy(0, 1); + Kokkos::parallel_for("Fill 2D FE matrix: corner points", policy, *this); } } - - KOKKOS_INLINE_FUNCTION - void operator() (const interiorFDTag&, const size_type idx) const { - ordinal_type i, j; - - // Compute row index - j = idx / (nx - 2); - i = idx % (nx - 2); - const ordinal_type rowIdx = (j + 1)*nx + i + 1; - - // Compute rowOffset - const size_type rowOffset = j*numEntriesPerGridRow + numEntriesBottomRow - + (i + 1)*interiorStencilLength + edgeStencilLength; - - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 5) = rowIdx - nx; - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + 1; - columns(rowOffset - 1) = rowIdx + nx; - + } + + KOKKOS_INLINE_FUNCTION + void operator()(const interiorFDTag&, const size_type idx) const { + ordinal_type i, j; + + // Compute row index + j = idx / (nx - 2); + i = idx % (nx - 2); + const ordinal_type rowIdx = (j + 1) * nx + i + 1; + + // Compute rowOffset + const size_type rowOffset = j * numEntriesPerGridRow + numEntriesBottomRow + + (i + 1) * interiorStencilLength + + edgeStencilLength; + + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 5) = rowIdx - nx; + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + 1; + columns(rowOffset - 1) = rowIdx + nx; + + // Fill values + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 4.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } + + KOKKOS_INLINE_FUNCTION + void operator()(const xEdgeFDTag&, const size_type idx) const { + /***************/ + /* Bottom edge */ + /***************/ + ordinal_type rowIdx = idx + 1; + size_type rowOffset = (idx + 1) * edgeStencilLength + cornerStencilLength; + + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + 1; + columns(rowOffset - 1) = rowIdx + nx; + if (bottomBC == 1) { + // Fill values + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { // Fill values - values(rowOffset - 5) = -1.0; values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 4.0; + values(rowOffset - 3) = 3.0; values(rowOffset - 2) = -1.0; values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const xEdgeFDTag&, const size_type idx) const { - - /***************/ - /* Bottom edge */ - /***************/ - ordinal_type rowIdx = idx + 1; - size_type rowOffset = (idx + 1)*edgeStencilLength + cornerStencilLength; - - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + 1; - columns(rowOffset - 1) = rowIdx + nx; - if(bottomBC == 1) { - // Fill values - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 3.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - /************/ - /* Top edge */ - /************/ - rowIdx = (ny - 1)*nx + idx + 1; - rowOffset = (ny - 2)*numEntriesPerGridRow + numEntriesBottomRow - + (idx + 1)*edgeStencilLength + cornerStencilLength; - - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 4) = rowIdx - nx; - columns(rowOffset - 3) = rowIdx - 1; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + 1; - if(topBC == 1) { - // Fill values - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 1.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 3.0; - values(rowOffset - 1) = -1.0; - } + /************/ + /* Top edge */ + /************/ + rowIdx = (ny - 1) * nx + idx + 1; + rowOffset = (ny - 2) * numEntriesPerGridRow + numEntriesBottomRow + + (idx + 1) * edgeStencilLength + cornerStencilLength; + + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 4) = rowIdx - nx; + columns(rowOffset - 3) = rowIdx - 1; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + 1; + if (topBC == 1) { + // Fill values + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 1.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 3.0; + values(rowOffset - 1) = -1.0; } - - KOKKOS_INLINE_FUNCTION - void operator() (const yEdgeFDTag&, const size_type idx) const { - - /*************/ - /* Left edge */ - /*************/ - ordinal_type rowIdx = (idx + 1)*nx; - size_type rowOffset = idx*numEntriesPerGridRow + numEntriesBottomRow - + edgeStencilLength; - - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 4) = rowIdx - nx; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + 1; - columns(rowOffset - 1) = rowIdx + nx; - if(leftBC == 1) { - // Fill values - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 3.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - /**************/ - /* Right edge */ - /**************/ - rowIdx = (idx + 2)*nx - 1; - rowOffset = (idx + 1)*numEntriesPerGridRow + numEntriesBottomRow; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 4) = rowIdx - nx; - columns(rowOffset - 3) = rowIdx - 1; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + nx; - if(rightBC == 1) { - // Fill values - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 1.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 3.0; - values(rowOffset - 1) = -1.0; - } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const yEdgeFDTag&, const size_type idx) const { + /*************/ + /* Left edge */ + /*************/ + ordinal_type rowIdx = (idx + 1) * nx; + size_type rowOffset = + idx * numEntriesPerGridRow + numEntriesBottomRow + edgeStencilLength; + + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 4) = rowIdx - nx; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + 1; + columns(rowOffset - 1) = rowIdx + nx; + if (leftBC == 1) { + // Fill values + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 3.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const cornerFDTag&, const size_type /*idx*/) const { - // Bottom-left corner - ordinal_type rowIdx = 0; - size_type rowOffset = cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + 1; - columns(rowOffset - 1) = rowIdx + nx; - if(bottomBC == 1 || leftBC == 1) { - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - values(rowOffset - 3) = 2.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - // Bottom-right corner - rowIdx = nx - 1; - rowOffset = numEntriesBottomRow; - rowmap(rowIdx + 1) = rowOffset; - - columns(rowOffset - 3) = rowIdx - 1; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + nx; - if(bottomBC == 1 || rightBC == 1) { - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 2.0; - values(rowOffset - 1) = -1.0; - } - - // Top-left corner - rowIdx = (ny - 1)*nx; - rowOffset = (ny - 2)*numEntriesPerGridRow + numEntriesBottomRow - + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - columns(rowOffset - 3) = rowIdx - nx; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + 1; - if(topBC == 1 || leftBC == 1) { - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 1.0; - values(rowOffset - 1) = 0.0; - } else { - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 2.0; - values(rowOffset - 1) = -1.0; - } - - // Top-right corner - rowIdx = ny*nx - 1; - rowOffset = numEntries; - rowmap(rowIdx + 1) = rowOffset; - - columns(rowOffset - 3) = rowIdx - nx; - columns(rowOffset - 2) = rowIdx - 1; - columns(rowOffset - 1) = rowIdx; - if(topBC == 1 || rightBC == 1) { - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 1.0; - } else { - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = 2.0; - } + /**************/ + /* Right edge */ + /**************/ + rowIdx = (idx + 2) * nx - 1; + rowOffset = (idx + 1) * numEntriesPerGridRow + numEntriesBottomRow; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 4) = rowIdx - nx; + columns(rowOffset - 3) = rowIdx - 1; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + nx; + if (rightBC == 1) { + // Fill values + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 1.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 3.0; + values(rowOffset - 1) = -1.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const cornerFDTag&, const size_type /*idx*/) const { + // Bottom-left corner + ordinal_type rowIdx = 0; + size_type rowOffset = cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + 1; + columns(rowOffset - 1) = rowIdx + nx; + if (bottomBC == 1 || leftBC == 1) { + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + values(rowOffset - 3) = 2.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const interiorFETag&, const size_type idx) const { - ordinal_type i, j; - - // Compute row index - j = idx / (nx - 2); - i = idx % (nx - 2); - const ordinal_type rowIdx = (j + 1)*nx + i + 1; + // Bottom-right corner + rowIdx = nx - 1; + rowOffset = numEntriesBottomRow; + rowmap(rowIdx + 1) = rowOffset; + + columns(rowOffset - 3) = rowIdx - 1; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + nx; + if (bottomBC == 1 || rightBC == 1) { + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 2.0; + values(rowOffset - 1) = -1.0; + } - // Compute rowOffset - const size_type rowOffset = j*numEntriesPerGridRow + numEntriesBottomRow - + (i + 1)*interiorStencilLength + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; + // Top-left corner + rowIdx = (ny - 1) * nx; + rowOffset = (ny - 2) * numEntriesPerGridRow + numEntriesBottomRow + + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + columns(rowOffset - 3) = rowIdx - nx; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + 1; + if (topBC == 1 || leftBC == 1) { + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 1.0; + values(rowOffset - 1) = 0.0; + } else { + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 2.0; + values(rowOffset - 1) = -1.0; + } - // Fill column indices - columns(rowOffset - 9) = rowIdx - nx - 1; - columns(rowOffset - 8) = rowIdx - nx; - columns(rowOffset - 7) = rowIdx - nx + 1; - columns(rowOffset - 6) = rowIdx - 1; - columns(rowOffset - 5) = rowIdx; - columns(rowOffset - 4) = rowIdx + 1; - columns(rowOffset - 3) = rowIdx + nx - 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + nx + 1; + // Top-right corner + rowIdx = ny * nx - 1; + rowOffset = numEntries; + rowmap(rowIdx + 1) = rowOffset; + + columns(rowOffset - 3) = rowIdx - nx; + columns(rowOffset - 2) = rowIdx - 1; + columns(rowOffset - 1) = rowIdx; + if (topBC == 1 || rightBC == 1) { + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 1.0; + } else { + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = 2.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const interiorFETag&, const size_type idx) const { + ordinal_type i, j; + + // Compute row index + j = idx / (nx - 2); + i = idx % (nx - 2); + const ordinal_type rowIdx = (j + 1) * nx + i + 1; + + // Compute rowOffset + const size_type rowOffset = j * numEntriesPerGridRow + numEntriesBottomRow + + (i + 1) * interiorStencilLength + + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 9) = rowIdx - nx - 1; + columns(rowOffset - 8) = rowIdx - nx; + columns(rowOffset - 7) = rowIdx - nx + 1; + columns(rowOffset - 6) = rowIdx - 1; + columns(rowOffset - 5) = rowIdx; + columns(rowOffset - 4) = rowIdx + 1; + columns(rowOffset - 3) = rowIdx + nx - 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + nx + 1; + + // Fill values + values(rowOffset - 9) = -2.0; + values(rowOffset - 8) = -2.0; + values(rowOffset - 7) = -2.0; + values(rowOffset - 6) = -2.0; + values(rowOffset - 5) = 16.0; + values(rowOffset - 4) = -2.0; + values(rowOffset - 3) = -2.0; + values(rowOffset - 2) = -2.0; + values(rowOffset - 1) = -2.0; + } + + KOKKOS_INLINE_FUNCTION + void operator()(const xEdgeFETag&, const size_type idx) const { + /***************/ + /* Bottom edge */ + /***************/ + ordinal_type rowIdx = idx + 1; + size_type rowOffset = (idx + 1) * edgeStencilLength + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 6) = rowIdx - 1; + columns(rowOffset - 5) = rowIdx; + columns(rowOffset - 4) = rowIdx + 1; + columns(rowOffset - 3) = rowIdx + nx - 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + nx + 1; + if (bottomBC == 1) { + // Fill values + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 1.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = 8.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -2.0; + values(rowOffset - 2) = -2.0; + values(rowOffset - 1) = -2.0; + } + /************/ + /* Top edge */ + /************/ + rowIdx = (ny - 1) * nx + idx + 1; + rowOffset = (ny - 2) * numEntriesPerGridRow + numEntriesBottomRow + + (idx + 1) * edgeStencilLength + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 6) = rowIdx - nx - 1; + columns(rowOffset - 5) = rowIdx - nx; + columns(rowOffset - 4) = rowIdx - nx + 1; + columns(rowOffset - 3) = rowIdx - 1; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + 1; + if (topBC == 1) { + // Fill values + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 1.0; + values(rowOffset - 1) = 0.0; + } else { // Fill values - values(rowOffset - 9) = -2.0; - values(rowOffset - 8) = -2.0; - values(rowOffset - 7) = -2.0; values(rowOffset - 6) = -2.0; - values(rowOffset - 5) = 16.0; + values(rowOffset - 5) = -2.0; values(rowOffset - 4) = -2.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 8.0; + values(rowOffset - 1) = -1.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const yEdgeFETag&, const size_type idx) const { + /*************/ + /* Left edge */ + /*************/ + ordinal_type rowIdx = (idx + 1) * nx; + size_type rowOffset = + idx * numEntriesPerGridRow + numEntriesBottomRow + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 6) = rowIdx - nx; + columns(rowOffset - 5) = rowIdx - nx + 1; + columns(rowOffset - 4) = rowIdx; + columns(rowOffset - 3) = rowIdx + 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + nx + 1; + if (leftBC == 1) { + // Fill values + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -2.0; + values(rowOffset - 4) = 8.0; values(rowOffset - 3) = -2.0; - values(rowOffset - 2) = -2.0; + values(rowOffset - 2) = -1.0; values(rowOffset - 1) = -2.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const xEdgeFETag&, const size_type idx) const { - - /***************/ - /* Bottom edge */ - /***************/ - ordinal_type rowIdx = idx + 1; - size_type rowOffset = (idx + 1)*edgeStencilLength + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 6) = rowIdx - 1; - columns(rowOffset - 5) = rowIdx; - columns(rowOffset - 4) = rowIdx + 1; - columns(rowOffset - 3) = rowIdx + nx - 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + nx + 1; - if(bottomBC == 1) { - // Fill values - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 1.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = 8.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -2.0; - values(rowOffset - 2) = -2.0; - values(rowOffset - 1) = -2.0; - } - - /************/ - /* Top edge */ - /************/ - rowIdx = (ny - 1)*nx + idx + 1; - rowOffset = (ny - 2)*numEntriesPerGridRow + numEntriesBottomRow - + (idx + 1)*edgeStencilLength + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 6) = rowIdx - nx - 1; - columns(rowOffset - 5) = rowIdx - nx; - columns(rowOffset - 4) = rowIdx - nx + 1; - columns(rowOffset - 3) = rowIdx - 1; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + 1; - if(topBC == 1) { - // Fill values - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 1.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 6) = -2.0; - values(rowOffset - 5) = -2.0; - values(rowOffset - 4) = -2.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 8.0; - values(rowOffset - 1) = -1.0; - } + /**************/ + /* Right edge */ + /**************/ + rowIdx = (idx + 2) * nx - 1; + rowOffset = (idx + 1) * numEntriesPerGridRow + numEntriesBottomRow; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 6) = rowIdx - nx - 1; + columns(rowOffset - 5) = rowIdx - nx; + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + nx - 1; + columns(rowOffset - 1) = rowIdx + nx; + if (rightBC == 1) { + // Fill values + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 6) = -2.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -2.0; + values(rowOffset - 3) = 8.0; + values(rowOffset - 2) = -2.0; + values(rowOffset - 1) = -1.0; } - - KOKKOS_INLINE_FUNCTION - void operator() (const yEdgeFETag&, const size_type idx) const { - - /*************/ - /* Left edge */ - /*************/ - ordinal_type rowIdx = (idx + 1)*nx; - size_type rowOffset = idx*numEntriesPerGridRow + numEntriesBottomRow - + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 6) = rowIdx - nx; - columns(rowOffset - 5) = rowIdx - nx + 1; - columns(rowOffset - 4) = rowIdx; - columns(rowOffset - 3) = rowIdx + 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + nx + 1; - if(leftBC == 1) { - // Fill values - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -2.0; - values(rowOffset - 4) = 8.0; - values(rowOffset - 3) = -2.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -2.0; - } - - /**************/ - /* Right edge */ - /**************/ - rowIdx = (idx + 2)*nx - 1; - rowOffset = (idx + 1)*numEntriesPerGridRow + numEntriesBottomRow; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 6) = rowIdx - nx - 1; - columns(rowOffset - 5) = rowIdx - nx; - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + nx - 1; - columns(rowOffset - 1) = rowIdx + nx; - if(rightBC == 1) { - // Fill values - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 6) = -2.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = -2.0; - values(rowOffset - 3) = 8.0; - values(rowOffset - 2) = -2.0; - values(rowOffset - 1) = -1.0; - } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const cornerFETag&, const size_type /*idx*/) const { + // Bottom-left corner + ordinal_type rowIdx = 0; + size_type rowOffset = cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + columns(rowOffset - 4) = rowIdx; + columns(rowOffset - 3) = rowIdx + 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + nx + 1; + if (bottomBC == 1 || leftBC == 1) { + values(rowOffset - 4) = 1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + values(rowOffset - 4) = 4.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -2.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const cornerFETag&, const size_type /*idx*/) const { - // Bottom-left corner - ordinal_type rowIdx = 0; - size_type rowOffset = cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - columns(rowOffset - 4) = rowIdx; - columns(rowOffset - 3) = rowIdx + 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + nx + 1; - if(bottomBC == 1 || leftBC == 1) { - values(rowOffset - 4) = 1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - values(rowOffset - 4) = 4.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -2.0; - } - - // Bottom-right corner - rowIdx = nx - 1; - rowOffset = numEntriesBottomRow; - rowmap(rowIdx + 1) = rowOffset; - - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + nx - 1; - columns(rowOffset - 1) = rowIdx + nx; - if(bottomBC == 1 || rightBC == 1) { - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 4.0; - values(rowOffset - 2) = -2.0; - values(rowOffset - 1) = -1.0; - } - - // Top-left corner - rowIdx = (ny - 1)*nx; - rowOffset = (ny - 2)*numEntriesPerGridRow + numEntriesBottomRow - + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - columns(rowOffset - 4) = rowIdx - nx; - columns(rowOffset - 3) = rowIdx - nx + 1; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + 1; - if(topBC == 1 || leftBC == 1) { - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 1.0; - values(rowOffset - 1) = 0.0; - } else { - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -2.0; - values(rowOffset - 2) = 4.0; - values(rowOffset - 1) = -1.0; - } - - // Top-right corner - rowIdx = ny*nx - 1; - rowOffset = numEntries; - rowmap(rowIdx + 1) = rowOffset; - - columns(rowOffset - 4) = rowIdx - nx - 1; - columns(rowOffset - 3) = rowIdx - nx; - columns(rowOffset - 2) = rowIdx - 1; - columns(rowOffset - 1) = rowIdx; - if(topBC == 1 || rightBC == 1) { - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 1.0; - } else { - values(rowOffset - 4) = -2.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = 4.0; - } + // Bottom-right corner + rowIdx = nx - 1; + rowOffset = numEntriesBottomRow; + rowmap(rowIdx + 1) = rowOffset; + + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + nx - 1; + columns(rowOffset - 1) = rowIdx + nx; + if (bottomBC == 1 || rightBC == 1) { + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 4.0; + values(rowOffset - 2) = -2.0; + values(rowOffset - 1) = -1.0; } - }; - template - CrsMatrix_t generate_structured_matrix2D(const std::string stencil, - const mat_structure& structure) { - - typedef typename CrsMatrix_t::StaticCrsGraphType graph_t; - typedef typename CrsMatrix_t::row_map_type::non_const_type row_map_view_t; - typedef typename CrsMatrix_t::index_type::non_const_type cols_view_t; - typedef typename CrsMatrix_t::values_type::non_const_type scalar_view_t; - typedef typename CrsMatrix_t::non_const_size_type size_type; - typedef typename CrsMatrix_t::non_const_ordinal_type ordinal_type; + // Top-left corner + rowIdx = (ny - 1) * nx; + rowOffset = (ny - 2) * numEntriesPerGridRow + numEntriesBottomRow + + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + columns(rowOffset - 4) = rowIdx - nx; + columns(rowOffset - 3) = rowIdx - nx + 1; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + 1; + if (topBC == 1 || leftBC == 1) { + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 1.0; + values(rowOffset - 1) = 0.0; + } else { + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -2.0; + values(rowOffset - 2) = 4.0; + values(rowOffset - 1) = -1.0; + } - int stencil_type = 0; - if (stencil == "FD") { - stencil_type = FD; - } else if (stencil == "FE") { - stencil_type = FE; + // Top-right corner + rowIdx = ny * nx - 1; + rowOffset = numEntries; + rowmap(rowIdx + 1) = rowOffset; + + columns(rowOffset - 4) = rowIdx - nx - 1; + columns(rowOffset - 3) = rowIdx - nx; + columns(rowOffset - 2) = rowIdx - 1; + columns(rowOffset - 1) = rowIdx; + if (topBC == 1 || rightBC == 1) { + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 1.0; } else { - std::ostringstream os; - os << "Test::generate_structured_matrix2D only accepts stencil: FD and FEM, you passed: " - << stencil <<" !" << std::endl; - Kokkos::Impl::throw_runtime_exception (os.str ()); - } - - // Extract geometric data - const ordinal_type nx = structure(0,0); - const ordinal_type ny = structure(1,0); - const ordinal_type numNodes = ny*nx; - const ordinal_type leftBC = structure(0,1); - const ordinal_type rightBC = structure(0,2); - const ordinal_type bottomBC = structure(1,1); - const ordinal_type topBC = structure(1,2); - const ordinal_type numInterior = (nx - 2)*(ny - 2); - const ordinal_type numEdge = 2*(nx - 2) + 2*(ny - 2); - const ordinal_type numCorner = 4; - ordinal_type interiorStencilLength = 0, edgeStencilLength = 0, cornerStencilLength = 0; - - if(stencil_type == FD) { - interiorStencilLength = 5; - edgeStencilLength = 4; - cornerStencilLength = 3; - } else if(stencil_type == FE) { - interiorStencilLength = 9; - edgeStencilLength = 6; + values(rowOffset - 4) = -2.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = 4.0; + } + } +}; + +template +CrsMatrix_t generate_structured_matrix2D(const std::string stencil, + const mat_structure& structure) { + typedef typename CrsMatrix_t::StaticCrsGraphType graph_t; + typedef typename CrsMatrix_t::row_map_type::non_const_type row_map_view_t; + typedef typename CrsMatrix_t::index_type::non_const_type cols_view_t; + typedef typename CrsMatrix_t::values_type::non_const_type scalar_view_t; + typedef typename CrsMatrix_t::non_const_size_type size_type; + typedef typename CrsMatrix_t::non_const_ordinal_type ordinal_type; + + int stencil_type = 0; + if (stencil == "FD") { + stencil_type = FD; + } else if (stencil == "FE") { + stencil_type = FE; + } else { + std::ostringstream os; + os << "Test::generate_structured_matrix2D only accepts stencil: FD and " + "FEM, you passed: " + << stencil << " !" << std::endl; + KokkosKernels::Impl::throw_runtime_exception(os.str()); + } + + // Extract geometric data + const ordinal_type nx = structure(0, 0); + const ordinal_type ny = structure(1, 0); + const ordinal_type numNodes = ny * nx; + const ordinal_type leftBC = structure(0, 1); + const ordinal_type rightBC = structure(0, 2); + const ordinal_type bottomBC = structure(1, 1); + const ordinal_type topBC = structure(1, 2); + const ordinal_type numInterior = (nx - 2) * (ny - 2); + const ordinal_type numEdge = 2 * (nx - 2) + 2 * (ny - 2); + const ordinal_type numCorner = 4; + ordinal_type interiorStencilLength = 0, edgeStencilLength = 0, + cornerStencilLength = 0; + + if (stencil_type == FD) { + interiorStencilLength = 5; + edgeStencilLength = 4; + cornerStencilLength = 3; + } else if (stencil_type == FE) { + interiorStencilLength = 9; + edgeStencilLength = 6; + cornerStencilLength = 4; + } + + const size_type numEntries = numInterior * interiorStencilLength + + numEdge * edgeStencilLength + + numCorner * cornerStencilLength; + + // Create matrix data + row_map_view_t rowmap_view("rowmap_view", numNodes + 1); + cols_view_t columns_view("colsmap_view", numEntries); + scalar_view_t values_view("values_view", numEntries); + + fill_2D_matrix_functor fill_2D_matrix( + stencil_type, nx, ny, leftBC, rightBC, bottomBC, topBC, rowmap_view, + columns_view, values_view); + + fill_2D_matrix.compute(); + + graph_t static_graph(columns_view, rowmap_view); + std::string name; + if (stencil_type == FD) { + name = "CrsMatrixFD"; + } else if (stencil_type == FE) { + name = "CrsMatrixFE"; + } + + return CrsMatrix_t(name, numNodes, values_view, static_graph); + +} // generate_structured_matrix2D + +template +struct fill_3D_matrix_functor { + // Define types used by the CrsMatrix + typedef typename CrsMatrix_t::execution_space execution_space; + typedef typename CrsMatrix_t::row_map_type::non_const_type row_map_view_t; + typedef typename CrsMatrix_t::index_type::non_const_type cols_view_t; + typedef typename CrsMatrix_t::values_type::non_const_type scalar_view_t; + typedef typename CrsMatrix_t::non_const_size_type size_type; + typedef typename CrsMatrix_t::non_const_ordinal_type ordinal_type; + + // Finite Difference dispatch tags + struct interiorFDTag {}; + + struct xFaceFDTag {}; + struct yFaceFDTag {}; + struct zFaceFDTag {}; + + struct xEdgeFDTag {}; + struct yEdgeFDTag {}; + struct zEdgeFDTag {}; + + struct cornerFDTag {}; + + // Finite Element dispatch tags + struct interiorFETag {}; + + struct xFaceFETag {}; + struct yFaceFETag {}; + struct zFaceFETag {}; + + struct xEdgeFETag {}; + struct yEdgeFETag {}; + struct zEdgeFETag {}; + + struct cornerFETag {}; + + // Internal variables and temporaries + const int stencil_type; + const ordinal_type nx, ny, nz; + const int leftBC, rightBC, frontBC, backBC, bottomBC, topBC; + + // Matrix views + row_map_view_t rowmap; + cols_view_t columns; + scalar_view_t values; + + size_type numEntries; + ordinal_type numInterior; + ordinal_type numXFace; + ordinal_type numYFace; + ordinal_type numZFace; + ordinal_type numXEdge; + ordinal_type numYEdge; + ordinal_type numZEdge; + + ordinal_type interiorStencilLength; + ordinal_type faceStencilLength; + ordinal_type edgeStencilLength; + ordinal_type cornerStencilLength; + ordinal_type numEntriesPerGridPlane; + ordinal_type numEntriesBottomPlane; + ordinal_type numEntriesPerGridRow; + ordinal_type numEntriesFrontRow; + ordinal_type numEntriesBottomFrontRow; + + fill_3D_matrix_functor(const int stencil_type_, const ordinal_type nx_, + const ordinal_type ny_, const ordinal_type nz_, + const int leftBC_, const int rightBC_, + const int frontBC_, const int backBC_, + const int bottomBC_, const int topBC_, + const row_map_view_t rowmap_, + const cols_view_t columns_, + const scalar_view_t values_) + : stencil_type(stencil_type_), + nx(nx_), + ny(ny_), + nz(nz_), + leftBC(leftBC_), + rightBC(rightBC_), + frontBC(frontBC_), + backBC(backBC_), + bottomBC(bottomBC_), + topBC(topBC_), + rowmap(rowmap_), + columns(columns_), + values(values_) { + if (stencil_type == FD) { + interiorStencilLength = 7; + faceStencilLength = 6; + edgeStencilLength = 5; cornerStencilLength = 4; + } else if (stencil_type == FE) { + interiorStencilLength = 27; + faceStencilLength = 18; + edgeStencilLength = 12; + cornerStencilLength = 8; } - const size_type numEntries = numInterior*interiorStencilLength - + numEdge*edgeStencilLength - + numCorner*cornerStencilLength; - - // Create matrix data - row_map_view_t rowmap_view ("rowmap_view", numNodes + 1); - cols_view_t columns_view("colsmap_view", numEntries); - scalar_view_t values_view ("values_view", numEntries); - - fill_2D_matrix_functor fill_2D_matrix(stencil_type, - nx, ny, - leftBC, rightBC, - bottomBC, topBC, - rowmap_view, - columns_view, - values_view); - - fill_2D_matrix.compute(); - - graph_t static_graph (columns_view, rowmap_view); - std::string name; - if(stencil_type == FD) { - name = "CrsMatrixFD"; - } else if(stencil_type == FE) { - name = "CrsMatrixFE"; - } - - return CrsMatrix_t(name, numNodes, values_view, static_graph); - - } // generate_structured_matrix2D - - template - struct fill_3D_matrix_functor { - - // Define types used by the CrsMatrix - typedef typename CrsMatrix_t::execution_space execution_space; - typedef typename CrsMatrix_t::row_map_type::non_const_type row_map_view_t; - typedef typename CrsMatrix_t::index_type::non_const_type cols_view_t; - typedef typename CrsMatrix_t::values_type::non_const_type scalar_view_t; - typedef typename CrsMatrix_t::non_const_size_type size_type; - typedef typename CrsMatrix_t::non_const_ordinal_type ordinal_type; - - // Finite Difference dispatch tags - struct interiorFDTag{}; - - struct xFaceFDTag{}; - struct yFaceFDTag{}; - struct zFaceFDTag{}; - - struct xEdgeFDTag{}; - struct yEdgeFDTag{}; - struct zEdgeFDTag{}; - - struct cornerFDTag{}; - - // Finite Element dispatch tags - struct interiorFETag{}; - - struct xFaceFETag{}; - struct yFaceFETag{}; - struct zFaceFETag{}; - - struct xEdgeFETag{}; - struct yEdgeFETag{}; - struct zEdgeFETag{}; - - struct cornerFETag{}; - - // Internal variables and temporaries - const int stencil_type; - const ordinal_type nx, ny, nz; - const int leftBC, rightBC, frontBC, backBC, bottomBC, topBC; - - // Matrix views - row_map_view_t rowmap; - cols_view_t columns; - scalar_view_t values; - - size_type numEntries; - ordinal_type numInterior; - ordinal_type numXFace; - ordinal_type numYFace; - ordinal_type numZFace; - ordinal_type numXEdge; - ordinal_type numYEdge; - ordinal_type numZEdge; - - ordinal_type interiorStencilLength; - ordinal_type faceStencilLength; - ordinal_type edgeStencilLength; - ordinal_type cornerStencilLength; - ordinal_type numEntriesPerGridPlane; - ordinal_type numEntriesBottomPlane; - ordinal_type numEntriesPerGridRow; - ordinal_type numEntriesFrontRow; - ordinal_type numEntriesBottomFrontRow; - - fill_3D_matrix_functor(const int stencil_type_, const ordinal_type nx_, - const ordinal_type ny_, const ordinal_type nz_, - const int leftBC_, const int rightBC_, - const int frontBC_, const int backBC_, - const int bottomBC_, const int topBC_, - const row_map_view_t rowmap_, const cols_view_t columns_, - const scalar_view_t values_) : - stencil_type(stencil_type_), nx(nx_), ny(ny_), nz(nz_), - leftBC(leftBC_), rightBC(rightBC_), frontBC(frontBC_), - backBC(backBC_), bottomBC(bottomBC_), topBC(topBC_), - rowmap(rowmap_), columns(columns_), values(values_) { - - if(stencil_type == FD) { - interiorStencilLength = 7; - faceStencilLength = 6; - edgeStencilLength = 5; - cornerStencilLength = 4; - } else if(stencil_type == FE) { - interiorStencilLength = 27; - faceStencilLength = 18; - edgeStencilLength = 12; - cornerStencilLength = 8; - } - - numInterior = (nx - 2)*(ny - 2)*(nz - 2); - numXFace = (ny - 2)*(nz - 2); - numYFace = (nx - 2)*(nz - 2); - numZFace = (nx - 2)*(ny - 2); - numXEdge = nx - 2; - numYEdge = ny - 2; - numZEdge = nz - 2; - - numEntries = numInterior*interiorStencilLength - + 2*(numXFace + numYFace + numZFace)*faceStencilLength - + 4*(numXEdge + numYEdge + numZEdge)*edgeStencilLength - + 8*cornerStencilLength; - numEntriesPerGridPlane = numZFace*interiorStencilLength - + 2*numXEdge*faceStencilLength - + 2*numYEdge*faceStencilLength - + 4*edgeStencilLength;; - numEntriesBottomPlane = numZFace*faceStencilLength - + 2*numXEdge*edgeStencilLength - + 2*numYEdge*edgeStencilLength - + 4*cornerStencilLength;; - numEntriesPerGridRow = numXEdge*interiorStencilLength - + 2*faceStencilLength; - numEntriesFrontRow = numXEdge*faceStencilLength - + 2*edgeStencilLength; - numEntriesBottomFrontRow = numXEdge*edgeStencilLength - + 2*cornerStencilLength; - } - - void compute() { - // Fill interior points - if(0 < numInterior) { - if(stencil_type == FD) { - Kokkos::RangePolicy policy(0, numInterior); - Kokkos::parallel_for("Fill 3D FD matrix: interior points", policy, *this); - } else if(stencil_type == FE) { - Kokkos::RangePolicy policy(0, numInterior); - Kokkos::parallel_for("Fill 3D FE matrix: interior points", policy, *this); - } - } - - // Fill x-faces - if(0 < numXFace) { - if(stencil_type == FD) { - Kokkos::RangePolicy policy(0, numXFace); - Kokkos::parallel_for("Fill 3D FD matrix: x-face points", policy, *this); - } else if(stencil_type == FE) { - Kokkos::RangePolicy policy(0, numXFace); - Kokkos::parallel_for("Fill 3D FE matrix: x-face points", policy, *this); - } + numInterior = (nx - 2) * (ny - 2) * (nz - 2); + numXFace = (ny - 2) * (nz - 2); + numYFace = (nx - 2) * (nz - 2); + numZFace = (nx - 2) * (ny - 2); + numXEdge = nx - 2; + numYEdge = ny - 2; + numZEdge = nz - 2; + + numEntries = numInterior * interiorStencilLength + + 2 * (numXFace + numYFace + numZFace) * faceStencilLength + + 4 * (numXEdge + numYEdge + numZEdge) * edgeStencilLength + + 8 * cornerStencilLength; + numEntriesPerGridPlane = + numZFace * interiorStencilLength + 2 * numXEdge * faceStencilLength + + 2 * numYEdge * faceStencilLength + 4 * edgeStencilLength; + ; + numEntriesBottomPlane = + numZFace * faceStencilLength + 2 * numXEdge * edgeStencilLength + + 2 * numYEdge * edgeStencilLength + 4 * cornerStencilLength; + ; + numEntriesPerGridRow = + numXEdge * interiorStencilLength + 2 * faceStencilLength; + numEntriesFrontRow = numXEdge * faceStencilLength + 2 * edgeStencilLength; + numEntriesBottomFrontRow = + numXEdge * edgeStencilLength + 2 * cornerStencilLength; + } + + void compute() { + // Fill interior points + if (0 < numInterior) { + if (stencil_type == FD) { + Kokkos::RangePolicy policy(0, + numInterior); + Kokkos::parallel_for("Fill 3D FD matrix: interior points", policy, + *this); + } else if (stencil_type == FE) { + Kokkos::RangePolicy policy(0, + numInterior); + Kokkos::parallel_for("Fill 3D FE matrix: interior points", policy, + *this); } + } - // Fill y-faces - if(0 < numYFace) { - if(stencil_type == FD) { - Kokkos::RangePolicy policy(0, numYFace); - Kokkos::parallel_for("Fill 3D FD matrix: y-face points", policy, *this); - } else if(stencil_type == FE) { - Kokkos::RangePolicy policy(0, numYFace); - Kokkos::parallel_for("Fill 3D FE matrix: y-face points", policy, *this); - } + // Fill x-faces + if (0 < numXFace) { + if (stencil_type == FD) { + Kokkos::RangePolicy policy(0, numXFace); + Kokkos::parallel_for("Fill 3D FD matrix: x-face points", policy, *this); + } else if (stencil_type == FE) { + Kokkos::RangePolicy policy(0, numXFace); + Kokkos::parallel_for("Fill 3D FE matrix: x-face points", policy, *this); } + } - // Fill z-faces - if(0 < numZFace) { - if(stencil_type == FD) { - Kokkos::RangePolicy policy(0, numZFace); - Kokkos::parallel_for("Fill 3D FD matrix: z-face points", policy, *this); - } else if(stencil_type == FE) { - Kokkos::RangePolicy policy(0, numZFace); - Kokkos::parallel_for("Fill 3D FE matrix: z-face points", policy, *this); - } + // Fill y-faces + if (0 < numYFace) { + if (stencil_type == FD) { + Kokkos::RangePolicy policy(0, numYFace); + Kokkos::parallel_for("Fill 3D FD matrix: y-face points", policy, *this); + } else if (stencil_type == FE) { + Kokkos::RangePolicy policy(0, numYFace); + Kokkos::parallel_for("Fill 3D FE matrix: y-face points", policy, *this); } + } - // Fill x-edges - if(0 < numXEdge) { - if(stencil_type == FD) { - Kokkos::RangePolicy policy(0, numXEdge); - Kokkos::parallel_for("Fill 3D FD matrix: x-edge points", policy, *this); - } else if(stencil_type == FE) { - Kokkos::RangePolicy policy(0, numXEdge); - Kokkos::parallel_for("Fill 3D FE matrix: x-edge points", policy, *this); - } + // Fill z-faces + if (0 < numZFace) { + if (stencil_type == FD) { + Kokkos::RangePolicy policy(0, numZFace); + Kokkos::parallel_for("Fill 3D FD matrix: z-face points", policy, *this); + } else if (stencil_type == FE) { + Kokkos::RangePolicy policy(0, numZFace); + Kokkos::parallel_for("Fill 3D FE matrix: z-face points", policy, *this); } + } - // Fill y-edges - if(0 < numYEdge) { - if(stencil_type == FD) { - Kokkos::RangePolicy policy(0, numYEdge); - Kokkos::parallel_for("Fill 3D FD matrix: y-edge points", policy, *this); - } else if(stencil_type == FE) { - Kokkos::RangePolicy policy(0, numYEdge); - Kokkos::parallel_for("Fill 3D FE matrix: y-edge points", policy, *this); - } + // Fill x-edges + if (0 < numXEdge) { + if (stencil_type == FD) { + Kokkos::RangePolicy policy(0, numXEdge); + Kokkos::parallel_for("Fill 3D FD matrix: x-edge points", policy, *this); + } else if (stencil_type == FE) { + Kokkos::RangePolicy policy(0, numXEdge); + Kokkos::parallel_for("Fill 3D FE matrix: x-edge points", policy, *this); } + } - // Fill z-edges - if(0 < numZEdge) { - if(stencil_type == FD) { - Kokkos::RangePolicy policy(0, numZEdge); - Kokkos::parallel_for("Fill 3D FD matrix: z-edge points", policy, *this); - } else if(stencil_type == FE) { - Kokkos::RangePolicy policy(0, numZEdge); - Kokkos::parallel_for("Fill 3D FE matrix: z-edge points", policy, *this); - } + // Fill y-edges + if (0 < numYEdge) { + if (stencil_type == FD) { + Kokkos::RangePolicy policy(0, numYEdge); + Kokkos::parallel_for("Fill 3D FD matrix: y-edge points", policy, *this); + } else if (stencil_type == FE) { + Kokkos::RangePolicy policy(0, numYEdge); + Kokkos::parallel_for("Fill 3D FE matrix: y-edge points", policy, *this); } + } - if(stencil_type == FD) { - Kokkos::RangePolicy policy(0, 1); - Kokkos::parallel_for("Fill 3D FD matrix: corner points", policy, *this); - } else if(stencil_type == FE) { - Kokkos::RangePolicy policy(0, 1); - Kokkos::parallel_for("Fill 3D FE matrix: corner points", policy, *this); + // Fill z-edges + if (0 < numZEdge) { + if (stencil_type == FD) { + Kokkos::RangePolicy policy(0, numZEdge); + Kokkos::parallel_for("Fill 3D FD matrix: z-edge points", policy, *this); + } else if (stencil_type == FE) { + Kokkos::RangePolicy policy(0, numZEdge); + Kokkos::parallel_for("Fill 3D FE matrix: z-edge points", policy, *this); } - } // compute() - - KOKKOS_INLINE_FUNCTION - void operator() (const interiorFDTag&, const size_type idx) const { - // Compute row index - const ordinal_type k = idx / ((ny - 2)*(nx - 2)); - const ordinal_type rem = idx % ((ny - 2)*(nx - 2)); - const ordinal_type j = rem / (nx - 2); - const ordinal_type i = rem % (nx - 2); - const ordinal_type rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i + 1; - - // Compute rowOffset - const size_type rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + j*numEntriesPerGridRow + numEntriesFrontRow - + (i + 1)*interiorStencilLength + faceStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 7) = rowIdx - ny*nx; - columns(rowOffset - 6) = rowIdx - nx; - columns(rowOffset - 5) = rowIdx - 1; - columns(rowOffset - 4) = rowIdx; - columns(rowOffset - 3) = rowIdx + 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx; + } + if (stencil_type == FD) { + Kokkos::RangePolicy policy(0, 1); + Kokkos::parallel_for("Fill 3D FD matrix: corner points", policy, *this); + } else if (stencil_type == FE) { + Kokkos::RangePolicy policy(0, 1); + Kokkos::parallel_for("Fill 3D FE matrix: corner points", policy, *this); + } + } // compute() + + KOKKOS_INLINE_FUNCTION + void operator()(const interiorFDTag&, const size_type idx) const { + // Compute row index + const ordinal_type k = idx / ((ny - 2) * (nx - 2)); + const ordinal_type rem = idx % ((ny - 2) * (nx - 2)); + const ordinal_type j = rem / (nx - 2); + const ordinal_type i = rem % (nx - 2); + const ordinal_type rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i + 1; + + // Compute rowOffset + const size_type rowOffset = + k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesPerGridRow + numEntriesFrontRow + + (i + 1) * interiorStencilLength + faceStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 7) = rowIdx - ny * nx; + columns(rowOffset - 6) = rowIdx - nx; + columns(rowOffset - 5) = rowIdx - 1; + columns(rowOffset - 4) = rowIdx; + columns(rowOffset - 3) = rowIdx + 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx; + + // Fill values + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = 6.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } + + KOKKOS_INLINE_FUNCTION + void operator()(const xFaceFDTag&, const size_type idx) const { + /*******************/ + /* x == 0 face */ + /*******************/ + // Compute row index + ordinal_type k = idx / (ny - 2); + ordinal_type j = idx % (ny - 2); + ordinal_type i = 0; + ordinal_type rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i; + + // Compute rowOffset + size_type rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesPerGridRow + numEntriesFrontRow + + faceStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 6) = rowIdx - ny * nx; + columns(rowOffset - 5) = rowIdx - nx; + columns(rowOffset - 4) = rowIdx; + columns(rowOffset - 3) = rowIdx + 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (leftBC == 1) { + // Fill values + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { // Fill values - values(rowOffset - 7) = -1.0; values(rowOffset - 6) = -1.0; values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 6.0; + values(rowOffset - 4) = 5.0; values(rowOffset - 3) = -1.0; values(rowOffset - 2) = -1.0; values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const xFaceFDTag&, const size_type idx) const { - /*******************/ - /* x == 0 face */ - /*******************/ - // Compute row index - ordinal_type k = idx / (ny - 2); - ordinal_type j = idx % (ny - 2); - ordinal_type i = 0; - ordinal_type rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i; - - // Compute rowOffset - size_type rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + j*numEntriesPerGridRow + numEntriesFrontRow + faceStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 6) = rowIdx - ny*nx; - columns(rowOffset - 5) = rowIdx - nx; - columns(rowOffset - 4) = rowIdx; - columns(rowOffset - 3) = rowIdx + 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(leftBC == 1) { - // Fill values - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 5.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - /********************/ - /* x == nx face */ - /********************/ - // Compute row index - k = idx / (ny - 2); - j = idx % (ny - 2); - i = nx - 1; - rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + (j + 1)*numEntriesPerGridRow + numEntriesFrontRow; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 6) = rowIdx - ny*nx; - columns(rowOffset - 5) = rowIdx - nx; - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(rightBC == 1) { - // Fill values - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 5.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } + /********************/ + /* x == nx face */ + /********************/ + // Compute row index + k = idx / (ny - 2); + j = idx % (ny - 2); + i = nx - 1; + rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + (j + 1) * numEntriesPerGridRow + numEntriesFrontRow; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 6) = rowIdx - ny * nx; + columns(rowOffset - 5) = rowIdx - nx; + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (rightBC == 1) { + // Fill values + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 5.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - - KOKKOS_INLINE_FUNCTION - void operator() (const yFaceFDTag&, const size_type idx) const { - /*******************/ - /* y == 0 face */ - /*******************/ - // Compute row index - ordinal_type k = idx / (nx - 2); - ordinal_type i = idx % (nx - 2); - ordinal_type rowIdx = (k + 1)*ny*nx + i + 1; - - // Compute rowOffset - size_type rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + (i + 1)*faceStencilLength + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 6) = rowIdx - ny*nx; - columns(rowOffset - 5) = rowIdx - 1; - columns(rowOffset - 4) = rowIdx; - columns(rowOffset - 3) = rowIdx + 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(frontBC == 1) { - // Fill values - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 5.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - /********************/ - /* y == ny face */ - /********************/ - // Compute row index - k = idx / (nx - 2); - ordinal_type j = ny - 2; - i = idx % (nx - 2); - rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i + 1; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + j*numEntriesPerGridRow + numEntriesFrontRow - + (i + 1)*faceStencilLength + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 6) = rowIdx - ny*nx; - columns(rowOffset - 5) = rowIdx - nx; - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + 1; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(backBC == 1) { - // Fill values - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 5.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const yFaceFDTag&, const size_type idx) const { + /*******************/ + /* y == 0 face */ + /*******************/ + // Compute row index + ordinal_type k = idx / (nx - 2); + ordinal_type i = idx % (nx - 2); + ordinal_type rowIdx = (k + 1) * ny * nx + i + 1; + + // Compute rowOffset + size_type rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + (i + 1) * faceStencilLength + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 6) = rowIdx - ny * nx; + columns(rowOffset - 5) = rowIdx - 1; + columns(rowOffset - 4) = rowIdx; + columns(rowOffset - 3) = rowIdx + 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (frontBC == 1) { + // Fill values + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = 5.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const zFaceFDTag&, const size_type idx) const { - /*******************/ - /* z == 0 face */ - /*******************/ - // Compute row index - ordinal_type j = idx / (nx - 2); - ordinal_type i = idx % (nx - 2); - ordinal_type rowIdx = (j + 1)*nx + i + 1; - - // Compute rowOffset - size_type rowOffset = j*numEntriesFrontRow + numEntriesBottomFrontRow - + (i + 1)*faceStencilLength + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 6) = rowIdx - nx; - columns(rowOffset - 5) = rowIdx - 1; - columns(rowOffset - 4) = rowIdx; - columns(rowOffset - 3) = rowIdx + 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(bottomBC == 1) { - // Fill values - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 5.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - /********************/ - /* z == nz face */ - /********************/ - // Compute row index - ordinal_type k = nz - 2; - j = idx / (nx - 2); - i = idx % (nx - 2); - rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i + 1; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + j*numEntriesFrontRow + numEntriesBottomFrontRow - + (i + 1)*faceStencilLength + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 6) = rowIdx - ny*nx; - columns(rowOffset - 5) = rowIdx - nx; - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + 1; - columns(rowOffset - 1) = rowIdx + nx; - if(topBC == 1) { - // Fill values - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 5.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } + /********************/ + /* y == ny face */ + /********************/ + // Compute row index + k = idx / (nx - 2); + ordinal_type j = ny - 2; + i = idx % (nx - 2); + rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i + 1; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesPerGridRow + numEntriesFrontRow + + (i + 1) * faceStencilLength + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 6) = rowIdx - ny * nx; + columns(rowOffset - 5) = rowIdx - nx; + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + 1; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (backBC == 1) { + // Fill values + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 5.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - - KOKKOS_INLINE_FUNCTION - void operator() (const xEdgeFDTag&, const size_type idx) const { - // Compute row index - ordinal_type i = idx; - ordinal_type rowIdx = i + 1; - - // Compute rowOffset - size_type rowOffset = (i + 1)*edgeStencilLength + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 5) = rowIdx - 1; - columns(rowOffset - 4) = rowIdx; - columns(rowOffset - 3) = rowIdx + 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(bottomBC == 1 || frontBC == 1) { - // Fill values - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 4.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - // Compute row index - ordinal_type j = ny - 2; - i = idx; - rowIdx = (j + 1)*nx + i + 1; - - // Compute rowOffset - rowOffset = j*numEntriesFrontRow + numEntriesBottomFrontRow - + (i + 1)*edgeStencilLength + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 5) = rowIdx - 1; - columns(rowOffset - 4) = rowIdx; - columns(rowOffset - 3) = rowIdx + 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(bottomBC == 1 || backBC == 1) { - // Fill values - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 4.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - // Compute row index - ordinal_type k = nz - 2; - i = idx; - rowIdx = (k + 1)*ny*nx + i + 1; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + (i + 1)*edgeStencilLength + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 5) = rowIdx - ny*nx; - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + 1; - columns(rowOffset - 1) = rowIdx + nx; - if(topBC == 1 || frontBC == 1) { - // Fill values - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 4.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - // Compute row index - k = nz - 2; - j = ny - 2; - i = idx; - rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i + 1; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + j*numEntriesFrontRow + numEntriesBottomFrontRow - + (i + 1)*edgeStencilLength + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 5) = rowIdx - ny*nx; - columns(rowOffset - 4) = rowIdx - nx; - columns(rowOffset - 3) = rowIdx - 1; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + 1; - if(topBC == 1 || backBC == 1) { - // Fill values - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 1.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 4.0; - values(rowOffset - 1) = -1.0; - } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const zFaceFDTag&, const size_type idx) const { + /*******************/ + /* z == 0 face */ + /*******************/ + // Compute row index + ordinal_type j = idx / (nx - 2); + ordinal_type i = idx % (nx - 2); + ordinal_type rowIdx = (j + 1) * nx + i + 1; + + // Compute rowOffset + size_type rowOffset = j * numEntriesFrontRow + numEntriesBottomFrontRow + + (i + 1) * faceStencilLength + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 6) = rowIdx - nx; + columns(rowOffset - 5) = rowIdx - 1; + columns(rowOffset - 4) = rowIdx; + columns(rowOffset - 3) = rowIdx + 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (bottomBC == 1) { + // Fill values + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = 5.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const yEdgeFDTag&, const size_type idx) const { - // Compute row index - ordinal_type j = idx; - ordinal_type rowIdx = (j + 1)*nx; - - // Compute rowOffset - size_type rowOffset = j*numEntriesFrontRow + numEntriesBottomFrontRow + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - - // Fill column indices - columns(rowOffset - 5) = rowIdx - nx; - columns(rowOffset - 4) = rowIdx; - columns(rowOffset - 3) = rowIdx + 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(bottomBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 4.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - // Compute row index - j = idx; - ordinal_type i = nx - 1; - rowIdx = (j + 1)*nx + i; - - // Compute rowOffset - rowOffset = (j + 1)*numEntriesFrontRow + numEntriesBottomFrontRow; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 5) = rowIdx - nx; - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(bottomBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 4.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - // Compute row index - ordinal_type k = nz - 2; - j = idx; - rowIdx = (k + 1)*ny*nx + (j + 1)*nx; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + j*numEntriesFrontRow + numEntriesBottomFrontRow - + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 5) = rowIdx - ny*nx; - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + 1; - columns(rowOffset - 1) = rowIdx + nx; - if(topBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 4.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - // Compute row index - k = nz - 2; - j = idx; - i = nx - 1; - rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + (j + 1)*numEntriesFrontRow + numEntriesBottomFrontRow; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 5) = rowIdx - ny*nx; - columns(rowOffset - 4) = rowIdx - nx; - columns(rowOffset - 3) = rowIdx - 1; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + nx; - if(topBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 1.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 4.0; - values(rowOffset - 1) = -1.0; - } + /********************/ + /* z == nz face */ + /********************/ + // Compute row index + ordinal_type k = nz - 2; + j = idx / (nx - 2); + i = idx % (nx - 2); + rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i + 1; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesFrontRow + numEntriesBottomFrontRow + + (i + 1) * faceStencilLength + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 6) = rowIdx - ny * nx; + columns(rowOffset - 5) = rowIdx - nx; + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + 1; + columns(rowOffset - 1) = rowIdx + nx; + if (topBC == 1) { + // Fill values + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 5.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const xEdgeFDTag&, const size_type idx) const { + // Compute row index + ordinal_type i = idx; + ordinal_type rowIdx = i + 1; + + // Compute rowOffset + size_type rowOffset = (i + 1) * edgeStencilLength + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 5) = rowIdx - 1; + columns(rowOffset - 4) = rowIdx; + columns(rowOffset - 3) = rowIdx + 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (bottomBC == 1 || frontBC == 1) { + // Fill values + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = 4.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const zEdgeFDTag&, const size_type idx) const { - // Compute row index - ordinal_type k = idx; - ordinal_type rowIdx = (k + 1)*ny*nx; - - // Compute rowOffset - size_type rowOffset = k*numEntriesPerGridPlane - + numEntriesBottomPlane + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 5) = rowIdx - ny*nx; - columns(rowOffset - 4) = rowIdx; - columns(rowOffset - 3) = rowIdx + 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(frontBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 4.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - // Compute row index - k = idx; - ordinal_type i = nx - 2; - rowIdx = (k + 1)*ny*nx + i + 1; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + i*faceStencilLength + 2*edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 5) = rowIdx - ny*nx; - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(frontBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 4.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - // Compute row index - k = idx; - ordinal_type j = ny - 2; - rowIdx = (k + 1)*ny*nx - + (j + 1)*nx; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane - + numEntriesBottomPlane - + j*numEntriesPerGridRow + numEntriesFrontRow - + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - - // Fill column indices - columns(rowOffset - 5) = rowIdx - ny*nx; - columns(rowOffset - 4) = rowIdx - nx; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + 1; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(backBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 4.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - // Compute row index - k = idx; - j = ny - 2; - i = nx - 2; - rowIdx = (k + 1)*ny*nx - + (j + 1)*nx + i + 1; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane - + numEntriesBottomPlane - + j*numEntriesPerGridRow + numEntriesFrontRow - + i*faceStencilLength + 2*edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 5) = rowIdx - ny*nx; - columns(rowOffset - 4) = rowIdx - nx; - columns(rowOffset - 3) = rowIdx - 1; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(backBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 1.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 4.0; - values(rowOffset - 1) = -1.0; - } + // Compute row index + ordinal_type j = ny - 2; + i = idx; + rowIdx = (j + 1) * nx + i + 1; + + // Compute rowOffset + rowOffset = j * numEntriesFrontRow + numEntriesBottomFrontRow + + (i + 1) * edgeStencilLength + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 5) = rowIdx - 1; + columns(rowOffset - 4) = rowIdx; + columns(rowOffset - 3) = rowIdx + 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (bottomBC == 1 || backBC == 1) { + // Fill values + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = 4.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const cornerFDTag&, const size_type /*idx*/) const { - // Bottom corners - ordinal_type rowIdx = 0; - size_type rowOffset = cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 4) = rowIdx; - columns(rowOffset - 3) = rowIdx + 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(bottomBC == 1 || frontBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 4) = 1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 4) = 3.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - rowIdx = nx - 1; - rowOffset = numEntriesBottomFrontRow; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(bottomBC == 1 || frontBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 3.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - rowIdx = (ny - 1)*nx; - rowOffset = (ny - 2)*numEntriesFrontRow - + numEntriesBottomFrontRow + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 4) = rowIdx - nx; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + 1; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(bottomBC == 1 || backBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 3.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - rowIdx = ny*nx - 1; - rowOffset = numEntriesBottomPlane; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 4) = rowIdx - nx; - columns(rowOffset - 3) = rowIdx - 1; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(bottomBC == 1 || backBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 1.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 3.0; - values(rowOffset - 1) = -1.0; - } - - rowIdx = (nz - 1)*ny*nx; - rowOffset = (nz - 2)*numEntriesPerGridPlane - + numEntriesBottomPlane + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 4) = rowIdx - ny*nx; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + 1; - columns(rowOffset - 1) = rowIdx + nx; - if(topBC == 1 || frontBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 3.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - rowIdx = (nz - 1)*ny*nx + nx - 1; - rowOffset = (nz - 2)*numEntriesPerGridPlane - + numEntriesBottomPlane + numEntriesBottomFrontRow; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 4) = rowIdx - ny*nx; - columns(rowOffset - 3) = rowIdx - 1; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + nx; - if(topBC == 1 || frontBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 1.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 3.0; - values(rowOffset - 1) = -1.0; - } - - rowIdx = nz*ny*nx - nx; - rowOffset = (nz - 2)*numEntriesPerGridPlane - + numEntriesBottomPlane + (ny - 2)*numEntriesFrontRow - + numEntriesBottomFrontRow + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 4) = rowIdx - ny*nx; - columns(rowOffset - 3) = rowIdx - nx; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + 1; - if(topBC == 1 || backBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 1.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 3.0; - values(rowOffset - 1) = -1.0; - } - - rowIdx = nz*ny*nx - 1; - rowOffset = numEntries; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 4) = rowIdx - ny*nx; - columns(rowOffset - 3) = rowIdx - nx; - columns(rowOffset - 2) = rowIdx - 1; - columns(rowOffset - 1) = rowIdx; - if(topBC == 1 || backBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 1.0; - } else { - // Fill values - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = 3.0; - } + // Compute row index + ordinal_type k = nz - 2; + i = idx; + rowIdx = (k + 1) * ny * nx + i + 1; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + (i + 1) * edgeStencilLength + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 5) = rowIdx - ny * nx; + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + 1; + columns(rowOffset - 1) = rowIdx + nx; + if (topBC == 1 || frontBC == 1) { + // Fill values + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 4.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const interiorFETag&, const size_type idx) const { - // Compute row index - const ordinal_type k = idx / ((ny - 2)*(nx - 2)); - const ordinal_type rem = idx % ((ny - 2)*(nx - 2)); - const ordinal_type j = rem / (nx - 2); - const ordinal_type i = rem % (nx - 2); - const ordinal_type rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i + 1; - - // Compute rowOffset - const size_type rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + j*numEntriesPerGridRow + numEntriesFrontRow - + (i + 1)*interiorStencilLength + faceStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 27) = rowIdx - ny*nx - nx - 1; - columns(rowOffset - 26) = rowIdx - ny*nx - nx; - columns(rowOffset - 25) = rowIdx - ny*nx - nx + 1; - columns(rowOffset - 24) = rowIdx - ny*nx - 1; - columns(rowOffset - 23) = rowIdx - ny*nx; - columns(rowOffset - 22) = rowIdx - ny*nx + 1; - columns(rowOffset - 21) = rowIdx - ny*nx + nx - 1; - columns(rowOffset - 20) = rowIdx - ny*nx + nx; - columns(rowOffset - 19) = rowIdx - ny*nx + nx + 1; - columns(rowOffset - 18) = rowIdx - nx - 1; - columns(rowOffset - 17) = rowIdx - nx; - columns(rowOffset - 16) = rowIdx - nx + 1; - columns(rowOffset - 15) = rowIdx - 1; - columns(rowOffset - 14) = rowIdx; - columns(rowOffset - 13) = rowIdx + 1; - columns(rowOffset - 12) = rowIdx + nx - 1; - columns(rowOffset - 11) = rowIdx + nx; - columns(rowOffset - 10) = rowIdx + nx + 1; - columns(rowOffset - 9) = rowIdx + nx*ny - nx - 1; - columns(rowOffset - 8) = rowIdx + nx*ny - nx; - columns(rowOffset - 7) = rowIdx + nx*ny - nx + 1; - columns(rowOffset - 6) = rowIdx + nx*ny - 1; - columns(rowOffset - 5) = rowIdx + nx*ny; - columns(rowOffset - 4) = rowIdx + nx*ny + 1; - columns(rowOffset - 3) = rowIdx + nx*ny + nx - 1; - columns(rowOffset - 2) = rowIdx + nx*ny + nx; - columns(rowOffset - 1) = rowIdx + nx*ny + nx + 1; - - // Fill values - values(rowOffset - 27) = -1.0; - values(rowOffset - 26) = -2.0; - values(rowOffset - 25) = -1.0; - values(rowOffset - 24) = -2.0; - values(rowOffset - 23) = 0.0; - values(rowOffset - 22) = -2.0; - values(rowOffset - 21) = -1.0; - values(rowOffset - 20) = -2.0; - values(rowOffset - 19) = -1.0; - values(rowOffset - 18) = -2.0; - values(rowOffset - 17) = 0.0; - values(rowOffset - 16) = -2.0; - values(rowOffset - 15) = 0.0; - values(rowOffset - 14) = 32.0; - values(rowOffset - 13) = 0.0; - values(rowOffset - 12) = -2.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = -2.0; - values(rowOffset - 9) = -1.0; - values(rowOffset - 8) = -2.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = -2.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = -2.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -2.0; - values(rowOffset - 1) = -1.0; - } - - KOKKOS_INLINE_FUNCTION - void operator() (const xFaceFETag&, const size_type idx) const { - /*******************/ - /* x == 0 face */ - /*******************/ - // Compute row index - ordinal_type k = idx / (ny - 2); - ordinal_type j = idx % (ny - 2); - ordinal_type i = 0; - ordinal_type rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i; - - // Compute rowOffset - size_type rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + j*numEntriesPerGridRow + numEntriesFrontRow + faceStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 18) = rowIdx - ny*nx - nx; - columns(rowOffset - 17) = rowIdx - ny*nx - nx + 1; - columns(rowOffset - 16) = rowIdx - ny*nx; - columns(rowOffset - 15) = rowIdx - ny*nx + 1; - columns(rowOffset - 14) = rowIdx - ny*nx + nx; - columns(rowOffset - 13) = rowIdx - ny*nx + nx + 1; - columns(rowOffset - 12) = rowIdx - nx; - columns(rowOffset - 11) = rowIdx - nx + 1; - columns(rowOffset - 10) = rowIdx; - columns(rowOffset - 9) = rowIdx + 1; - columns(rowOffset - 8) = rowIdx + nx; - columns(rowOffset - 7) = rowIdx + nx + 1; - columns(rowOffset - 6) = rowIdx + nx*ny - nx; - columns(rowOffset - 5) = rowIdx + nx*ny - nx + 1; - columns(rowOffset - 4) = rowIdx + nx*ny; - columns(rowOffset - 3) = rowIdx + nx*ny + 1; - columns(rowOffset - 2) = rowIdx + nx*ny + nx; - columns(rowOffset - 1) = rowIdx + nx*ny + nx + 1; - if(leftBC == 1) { - // Fill values - values(rowOffset - 18) = 0.0; - values(rowOffset - 17) = 0.0; - values(rowOffset - 16) = 0.0; - values(rowOffset - 15) = 0.0; - values(rowOffset - 14) = 0.0; - values(rowOffset - 13) = 0.0; - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 1.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 18) = -1.0; - values(rowOffset - 17) = -1.0; - values(rowOffset - 16) = 0.0; - values(rowOffset - 15) = -2.0; - values(rowOffset - 14) = -1.0; - values(rowOffset - 13) = -1.0; - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = -2.0; - values(rowOffset - 10) = 16.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = -2.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = -2.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - /********************/ - /* x == nx face */ - /********************/ - // Compute row index - k = idx / (ny - 2); - j = idx % (ny - 2); - i = nx - 1; - rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + (j + 1)*numEntriesPerGridRow + numEntriesFrontRow; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 18) = rowIdx - ny*nx - nx - 1; - columns(rowOffset - 17) = rowIdx - ny*nx - nx; - columns(rowOffset - 16) = rowIdx - ny*nx - 1; - columns(rowOffset - 15) = rowIdx - ny*nx; - columns(rowOffset - 14) = rowIdx - ny*nx + nx - 1; - columns(rowOffset - 13) = rowIdx - ny*nx + nx; - columns(rowOffset - 12) = rowIdx - nx - 1; - columns(rowOffset - 11) = rowIdx - nx; - columns(rowOffset - 10) = rowIdx - 1; - columns(rowOffset - 9) = rowIdx; - columns(rowOffset - 8) = rowIdx + nx - 1; - columns(rowOffset - 7) = rowIdx + nx; - columns(rowOffset - 6) = rowIdx + nx*ny - nx - 1; - columns(rowOffset - 5) = rowIdx + nx*ny - nx; - columns(rowOffset - 4) = rowIdx + nx*ny - 1; - columns(rowOffset - 3) = rowIdx + nx*ny; - columns(rowOffset - 2) = rowIdx + nx*ny + nx - 1; - columns(rowOffset - 1) = rowIdx + nx*ny + nx; - if(rightBC == 1) { - // Fill values - values(rowOffset - 18) = 0.0; - values(rowOffset - 17) = 0.0; - values(rowOffset - 16) = 0.0; - values(rowOffset - 15) = 0.0; - values(rowOffset - 14) = 0.0; - values(rowOffset - 13) = 0.0; - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 1.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 18) = -1.0; - values(rowOffset - 17) = -1.0; - values(rowOffset - 16) = -2.0; - values(rowOffset - 15) = 0.0; - values(rowOffset - 14) = -1.0; - values(rowOffset - 13) = -1.0; - values(rowOffset - 12) = -2.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 16.0; - values(rowOffset - 8) = -2.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = -2.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } + // Compute row index + k = nz - 2; + j = ny - 2; + i = idx; + rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i + 1; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesFrontRow + numEntriesBottomFrontRow + + (i + 1) * edgeStencilLength + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 5) = rowIdx - ny * nx; + columns(rowOffset - 4) = rowIdx - nx; + columns(rowOffset - 3) = rowIdx - 1; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + 1; + if (topBC == 1 || backBC == 1) { + // Fill values + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 1.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 4.0; + values(rowOffset - 1) = -1.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const yEdgeFDTag&, const size_type idx) const { + // Compute row index + ordinal_type j = idx; + ordinal_type rowIdx = (j + 1) * nx; + + // Compute rowOffset + size_type rowOffset = + j * numEntriesFrontRow + numEntriesBottomFrontRow + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 5) = rowIdx - nx; + columns(rowOffset - 4) = rowIdx; + columns(rowOffset - 3) = rowIdx + 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (bottomBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = 4.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const yFaceFETag&, const size_type idx) const { - /*******************/ - /* y == 0 face */ - /*******************/ - // Compute row index - ordinal_type k = idx / (nx - 2); - ordinal_type i = idx % (nx - 2); - ordinal_type rowIdx = (k + 1)*ny*nx + i + 1; - - // Compute rowOffset - size_type rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + (i + 1)*faceStencilLength + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 18) = rowIdx - ny*nx - 1; - columns(rowOffset - 17) = rowIdx - ny*nx; - columns(rowOffset - 16) = rowIdx - ny*nx + 1; - columns(rowOffset - 15) = rowIdx - ny*nx + nx - 1; - columns(rowOffset - 14) = rowIdx - ny*nx + nx; - columns(rowOffset - 13) = rowIdx - ny*nx + nx + 1; - columns(rowOffset - 12) = rowIdx - 1; - columns(rowOffset - 11) = rowIdx; - columns(rowOffset - 10) = rowIdx + 1; - columns(rowOffset - 9) = rowIdx + nx - 1; - columns(rowOffset - 8) = rowIdx + nx; - columns(rowOffset - 7) = rowIdx + nx + 1; - columns(rowOffset - 6) = rowIdx + nx*ny - 1; - columns(rowOffset - 5) = rowIdx + nx*ny; - columns(rowOffset - 4) = rowIdx + nx*ny + 1; - columns(rowOffset - 3) = rowIdx + nx*ny + nx - 1; - columns(rowOffset - 2) = rowIdx + nx*ny + nx; - columns(rowOffset - 1) = rowIdx + nx*ny + nx + 1; - if(frontBC == 1) { - // Fill values - values(rowOffset - 18) = 0.0; - values(rowOffset - 17) = 0.0; - values(rowOffset - 16) = 0.0; - values(rowOffset - 15) = 0.0; - values(rowOffset - 14) = 0.0; - values(rowOffset - 13) = 0.0; - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 1.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 18) = -1.0; - values(rowOffset - 17) = 0.0; - values(rowOffset - 16) = -1.0; - values(rowOffset - 15) = -1.0; - values(rowOffset - 14) = -2.0; - values(rowOffset - 13) = -1.0; - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 16.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = -2.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = -2.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -2.0; - values(rowOffset - 1) = -1.0; - } + // Compute row index + j = idx; + ordinal_type i = nx - 1; + rowIdx = (j + 1) * nx + i; + + // Compute rowOffset + rowOffset = (j + 1) * numEntriesFrontRow + numEntriesBottomFrontRow; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 5) = rowIdx - nx; + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (bottomBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 4.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } - /********************/ - /* y == ny face */ - /********************/ - // Compute row index - k = idx / (nx - 2); - ordinal_type j = ny - 2; - i = idx % (nx - 2); - rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i + 1; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + j*numEntriesPerGridRow + numEntriesFrontRow - + (i + 1)*faceStencilLength + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 18) = rowIdx - ny*nx - nx - 1; - columns(rowOffset - 17) = rowIdx - ny*nx - nx; - columns(rowOffset - 16) = rowIdx - ny*nx - 1; - columns(rowOffset - 15) = rowIdx - ny*nx - 1; - columns(rowOffset - 14) = rowIdx - ny*nx; - columns(rowOffset - 13) = rowIdx - ny*nx + 1; - columns(rowOffset - 12) = rowIdx - nx - 1; - columns(rowOffset - 11) = rowIdx - nx; - columns(rowOffset - 10) = rowIdx - nx + 1; - columns(rowOffset - 9) = rowIdx - 1; - columns(rowOffset - 8) = rowIdx; - columns(rowOffset - 7) = rowIdx + 1; - columns(rowOffset - 6) = rowIdx + nx*ny - nx - 1; - columns(rowOffset - 5) = rowIdx + nx*ny - nx; - columns(rowOffset - 4) = rowIdx + nx*ny - nx + 1; - columns(rowOffset - 3) = rowIdx + nx*ny - 1; - columns(rowOffset - 2) = rowIdx + nx*ny; - columns(rowOffset - 1) = rowIdx + nx*ny + 1; - if(backBC == 1) { - // Fill values - values(rowOffset - 18) = 0.0; - values(rowOffset - 17) = 0.0; - values(rowOffset - 16) = 0.0; - values(rowOffset - 15) = 0.0; - values(rowOffset - 14) = 0.0; - values(rowOffset - 13) = 0.0; - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 1.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 18) = -1.0; - values(rowOffset - 17) = -2.0; - values(rowOffset - 16) = -1.0; - values(rowOffset - 15) = -1.0; - values(rowOffset - 14) = 0.0; - values(rowOffset - 13) = -1.0; - values(rowOffset - 12) = -2.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = -2.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 16.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -2.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = -1.0; - } + // Compute row index + ordinal_type k = nz - 2; + j = idx; + rowIdx = (k + 1) * ny * nx + (j + 1) * nx; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesFrontRow + numEntriesBottomFrontRow + + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 5) = rowIdx - ny * nx; + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + 1; + columns(rowOffset - 1) = rowIdx + nx; + if (topBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 4.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const zFaceFETag&, const size_type idx) const { - /*******************/ - /* z == 0 face */ - /*******************/ - // Compute row index - ordinal_type j = idx / (nx - 2); - ordinal_type i = idx % (nx - 2); - ordinal_type rowIdx = (j + 1)*nx + i + 1; - - // Compute rowOffset - size_type rowOffset = j*numEntriesFrontRow + numEntriesBottomFrontRow - + (i + 1)*faceStencilLength + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 18) = rowIdx - nx - 1; - columns(rowOffset - 17) = rowIdx - nx; - columns(rowOffset - 16) = rowIdx - nx + 1; - columns(rowOffset - 15) = rowIdx - 1; - columns(rowOffset - 14) = rowIdx; - columns(rowOffset - 13) = rowIdx + 1; - columns(rowOffset - 12) = rowIdx + nx - 1; - columns(rowOffset - 11) = rowIdx + nx; - columns(rowOffset - 10) = rowIdx + nx + 1; - columns(rowOffset - 9) = rowIdx + nx*ny - nx - 1; - columns(rowOffset - 8) = rowIdx + nx*ny - nx; - columns(rowOffset - 7) = rowIdx + nx*ny - nx + 1; - columns(rowOffset - 6) = rowIdx + nx*ny - 1; - columns(rowOffset - 5) = rowIdx + nx*ny; - columns(rowOffset - 4) = rowIdx + nx*ny + 1; - columns(rowOffset - 3) = rowIdx + nx*ny + nx - 1; - columns(rowOffset - 2) = rowIdx + nx*ny + nx; - columns(rowOffset - 1) = rowIdx + nx*ny + nx + 1; - if(bottomBC == 1) { - // Fill values - values(rowOffset - 18) = 0.0; - values(rowOffset - 17) = 0.0; - values(rowOffset - 16) = 0.0; - values(rowOffset - 15) = 0.0; - values(rowOffset - 14) = 1.0; - values(rowOffset - 13) = 0.0; - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 18) = -1.0; - values(rowOffset - 17) = 0.0; - values(rowOffset - 16) = -1.0; - values(rowOffset - 15) = 0.0; - values(rowOffset - 14) = 16.0; - values(rowOffset - 13) = 0.0; - values(rowOffset - 12) = -1.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = -1.0; - values(rowOffset - 9) = -1.0; - values(rowOffset - 8) = -2.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = -2.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = -2.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -2.0; - values(rowOffset - 1) = -1.0; - } + // Compute row index + k = nz - 2; + j = idx; + i = nx - 1; + rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + (j + 1) * numEntriesFrontRow + numEntriesBottomFrontRow; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 5) = rowIdx - ny * nx; + columns(rowOffset - 4) = rowIdx - nx; + columns(rowOffset - 3) = rowIdx - 1; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + nx; + if (topBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 1.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 4.0; + values(rowOffset - 1) = -1.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const zEdgeFDTag&, const size_type idx) const { + // Compute row index + ordinal_type k = idx; + ordinal_type rowIdx = (k + 1) * ny * nx; + + // Compute rowOffset + size_type rowOffset = + k * numEntriesPerGridPlane + numEntriesBottomPlane + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 5) = rowIdx - ny * nx; + columns(rowOffset - 4) = rowIdx; + columns(rowOffset - 3) = rowIdx + 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (frontBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = 4.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } - /********************/ - /* z == nz face */ - /********************/ - // Compute row index - ordinal_type k = nz - 2; - j = idx / (nx - 2); - i = idx % (nx - 2); - rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i + 1; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + j*numEntriesFrontRow + numEntriesBottomFrontRow - + (i + 1)*faceStencilLength + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 18) = rowIdx - nx*ny - nx - 1; - columns(rowOffset - 17) = rowIdx - nx*ny - nx; - columns(rowOffset - 16) = rowIdx - nx*ny - nx + 1; - columns(rowOffset - 15) = rowIdx - nx*ny - 1; - columns(rowOffset - 14) = rowIdx - nx*ny; - columns(rowOffset - 13) = rowIdx - nx*ny + 1; - columns(rowOffset - 12) = rowIdx - nx*ny + nx - 1; - columns(rowOffset - 11) = rowIdx - nx*ny + nx; - columns(rowOffset - 10) = rowIdx - nx*ny + nx + 1; - columns(rowOffset - 9) = rowIdx - nx - 1; - columns(rowOffset - 8) = rowIdx - nx; - columns(rowOffset - 7) = rowIdx - nx + 1; - columns(rowOffset - 6) = rowIdx - 1; - columns(rowOffset - 5) = rowIdx; - columns(rowOffset - 4) = rowIdx + 1; - columns(rowOffset - 3) = rowIdx + nx - 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + nx + 1; - if(topBC == 1) { - // Fill values - values(rowOffset - 18) = 0.0; - values(rowOffset - 17) = 0.0; - values(rowOffset - 16) = 0.0; - values(rowOffset - 15) = 0.0; - values(rowOffset - 14) = 0.0; - values(rowOffset - 13) = 0.0; - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 1.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 18) = -1.0; - values(rowOffset - 17) = -2.0; - values(rowOffset - 16) = -1.0; - values(rowOffset - 15) = -2.0; - values(rowOffset - 14) = 0.0; - values(rowOffset - 13) = -2.0; - values(rowOffset - 12) = -1.0; - values(rowOffset - 11) = -2.0; - values(rowOffset - 10) = -1.0; - values(rowOffset - 9) = -1.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 16.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = -1.0; - } + // Compute row index + k = idx; + ordinal_type i = nx - 2; + rowIdx = (k + 1) * ny * nx + i + 1; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + i * faceStencilLength + 2 * edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 5) = rowIdx - ny * nx; + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (frontBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 4.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const xEdgeFETag&, const size_type idx) const { - - // Compute row index - ordinal_type i = idx; - ordinal_type rowIdx = i + 1; - - // Compute rowOffset - size_type rowOffset = (i + 1)*edgeStencilLength + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 12) = rowIdx - 1; - columns(rowOffset - 11) = rowIdx; - columns(rowOffset - 10) = rowIdx + 1; - columns(rowOffset - 9) = rowIdx + nx - 1; - columns(rowOffset - 8) = rowIdx + nx; - columns(rowOffset - 7) = rowIdx + nx + 1; - columns(rowOffset - 6) = rowIdx + nx*ny - 1; - columns(rowOffset - 5) = rowIdx + nx*ny; - columns(rowOffset - 4) = rowIdx + nx*ny + 1; - columns(rowOffset - 3) = rowIdx + nx*ny + nx - 1; - columns(rowOffset - 2) = rowIdx + nx*ny + nx; - columns(rowOffset - 1) = rowIdx + nx*ny + nx + 1; - if(bottomBC == 1 || frontBC == 1) { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 1.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 8.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = -1.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -2.0; - values(rowOffset - 1) = -1.0; - } + // Compute row index + k = idx; + ordinal_type j = ny - 2; + rowIdx = (k + 1) * ny * nx + (j + 1) * nx; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesPerGridRow + numEntriesFrontRow + + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 5) = rowIdx - ny * nx; + columns(rowOffset - 4) = rowIdx - nx; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + 1; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (backBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 4.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } - // Compute row index - ordinal_type j = ny - 2; - i = idx; - rowIdx = (j + 1)*nx + i + 1; - - // Compute rowOffset - rowOffset = j*numEntriesFrontRow + numEntriesBottomFrontRow - + (i + 1)*edgeStencilLength + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 12) = rowIdx - nx - 1; - columns(rowOffset - 11) = rowIdx - nx; - columns(rowOffset - 10) = rowIdx - nx + 1; - columns(rowOffset - 9) = rowIdx - 1; - columns(rowOffset - 8) = rowIdx; - columns(rowOffset - 7) = rowIdx + 1; - columns(rowOffset - 6) = rowIdx + nx*ny - nx - 1; - columns(rowOffset - 5) = rowIdx + nx*ny - nx; - columns(rowOffset - 4) = rowIdx + nx*ny - nx + 1; - columns(rowOffset - 3) = rowIdx + nx*ny - 1; - columns(rowOffset - 2) = rowIdx + nx*ny; - columns(rowOffset - 1) = rowIdx + nx*ny + 1; - if(bottomBC == 1 || backBC == 1) { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 1.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 12) = -1.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = -1.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 8.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -2.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = -1.0; - } + // Compute row index + k = idx; + j = ny - 2; + i = nx - 2; + rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i + 1; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesPerGridRow + numEntriesFrontRow + + i * faceStencilLength + 2 * edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 5) = rowIdx - ny * nx; + columns(rowOffset - 4) = rowIdx - nx; + columns(rowOffset - 3) = rowIdx - 1; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (backBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 1.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 4.0; + values(rowOffset - 1) = -1.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const cornerFDTag&, const size_type /*idx*/) const { + // Bottom corners + ordinal_type rowIdx = 0; + size_type rowOffset = cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 4) = rowIdx; + columns(rowOffset - 3) = rowIdx + 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (bottomBC == 1 || frontBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 4) = 1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 4) = 3.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } - // Compute row index - ordinal_type k = nz - 2; - i = idx; - rowIdx = (k + 1)*ny*nx + i + 1; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + (i + 1)*edgeStencilLength + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 12) = rowIdx - nx*ny - 1; - columns(rowOffset - 11) = rowIdx - nx*ny; - columns(rowOffset - 10) = rowIdx - nx*ny + 1; - columns(rowOffset - 9) = rowIdx - nx*ny + nx - 1; - columns(rowOffset - 8) = rowIdx - nx*ny + nx; - columns(rowOffset - 7) = rowIdx - nx*ny + nx + 1; - columns(rowOffset - 6) = rowIdx - 1; - columns(rowOffset - 5) = rowIdx; - columns(rowOffset - 4) = rowIdx + 1; - columns(rowOffset - 3) = rowIdx + nx - 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + nx + 1; - if(topBC == 1 || frontBC == 1) { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 1.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 12) = -1.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = -1.0; - values(rowOffset - 9) = -1.0; - values(rowOffset - 8) = -2.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 8.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = -1.0; - } + rowIdx = nx - 1; + rowOffset = numEntriesBottomFrontRow; + rowmap(rowIdx + 1) = rowOffset; - // Compute row index - k = nz - 2; - j = ny - 2; - i = idx; - rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i + 1; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + j*numEntriesFrontRow + numEntriesBottomFrontRow - + (i + 1)*edgeStencilLength + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 12) = rowIdx - nx*ny - nx - 1; - columns(rowOffset - 11) = rowIdx - nx*ny - nx; - columns(rowOffset - 10) = rowIdx - nx*ny - nx + 1; - columns(rowOffset - 9) = rowIdx - nx*ny - 1; - columns(rowOffset - 8) = rowIdx - nx*ny; - columns(rowOffset - 7) = rowIdx - nx*ny + 1; - columns(rowOffset - 6) = rowIdx - nx - 1; - columns(rowOffset - 5) = rowIdx - nx; - columns(rowOffset - 4) = rowIdx - nx + 1; - columns(rowOffset - 3) = rowIdx - 1; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + 1; - if(topBC == 1 || backBC == 1) { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 1.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 12) = -1.0; - values(rowOffset - 11) = -2.0; - values(rowOffset - 10) = -1.0; - values(rowOffset - 9) = -1.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 8.0; - values(rowOffset - 1) = 0.0; - } + // Fill column indices + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (bottomBC == 1 || frontBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 3.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const yEdgeFETag&, const size_type idx) const { - // Compute row index - ordinal_type j = idx; - ordinal_type rowIdx = (j + 1)*nx; - - // Compute rowOffset - size_type rowOffset = j*numEntriesFrontRow + numEntriesBottomFrontRow + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 12) = rowIdx - nx; - columns(rowOffset - 11) = rowIdx - nx + 1; - columns(rowOffset - 10) = rowIdx; - columns(rowOffset - 9) = rowIdx + 1; - columns(rowOffset - 8) = rowIdx + nx; - columns(rowOffset - 7) = rowIdx + nx + 1; - columns(rowOffset - 6) = rowIdx + nx*ny - nx; - columns(rowOffset - 5) = rowIdx + nx*ny - nx + 1; - columns(rowOffset - 4) = rowIdx + nx*ny; - columns(rowOffset - 3) = rowIdx + nx*ny + 1; - columns(rowOffset - 2) = rowIdx + nx*ny + nx; - columns(rowOffset - 1) = rowIdx + nx*ny + nx + 1; - if(bottomBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 1.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = -1.0; - values(rowOffset - 10) = 8.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = -2.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - // Compute row index - j = idx; - ordinal_type i = nx - 1; - rowIdx = (j + 1)*nx + i; - - // Compute rowOffset - rowOffset = (j + 1)*numEntriesFrontRow + numEntriesBottomFrontRow; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 12) = rowIdx - nx - 1; - columns(rowOffset - 11) = rowIdx - nx; - columns(rowOffset - 10) = rowIdx - 1; - columns(rowOffset - 9) = rowIdx; - columns(rowOffset - 8) = rowIdx + nx - 1; - columns(rowOffset - 7) = rowIdx + nx; - columns(rowOffset - 6) = rowIdx + nx*ny - nx - 1; - columns(rowOffset - 5) = rowIdx + nx*ny - nx; - columns(rowOffset - 4) = rowIdx + nx*ny - 1; - columns(rowOffset - 3) = rowIdx + nx*ny; - columns(rowOffset - 2) = rowIdx + nx*ny + nx - 1; - columns(rowOffset - 1) = rowIdx + nx*ny + nx; - if(bottomBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 12) = -1.0; - values(rowOffset - 11) = -1.0; - values(rowOffset - 10) = -2.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = -1.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 8.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = 0.0; - } + rowIdx = (ny - 1) * nx; + rowOffset = (ny - 2) * numEntriesFrontRow + numEntriesBottomFrontRow + + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 4) = rowIdx - nx; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + 1; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (bottomBC == 1 || backBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 3.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } - // Compute row index - ordinal_type k = nz - 2; - j = idx; - rowIdx = (k + 1)*ny*nx + (j + 1)*nx; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + j*numEntriesFrontRow + numEntriesBottomFrontRow - + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 12) = rowIdx - nx*ny - 1; - columns(rowOffset - 11) = rowIdx - nx*ny; - columns(rowOffset - 10) = rowIdx - nx*ny + 1; - columns(rowOffset - 9) = rowIdx - nx*ny + nx - 1; - columns(rowOffset - 8) = rowIdx - nx*ny + nx; - columns(rowOffset - 7) = rowIdx - nx*ny + nx + 1; - columns(rowOffset - 6) = rowIdx - 1; - columns(rowOffset - 5) = rowIdx; - columns(rowOffset - 4) = rowIdx + 1; - columns(rowOffset - 3) = rowIdx + nx - 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + nx + 1; - if(topBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 1.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 12) = -1.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = -1.0; - values(rowOffset - 9) = -1.0; - values(rowOffset - 8) = -2.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 8.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = -1.0; - } + rowIdx = ny * nx - 1; + rowOffset = numEntriesBottomPlane; + rowmap(rowIdx + 1) = rowOffset; - // Compute row index - k = nz - 2; - j = idx; - i = nx - 1; - rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + (j + 1)*numEntriesFrontRow + numEntriesBottomFrontRow; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 12) = rowIdx - nx*ny - nx - 1; - columns(rowOffset - 11) = rowIdx - nx*ny - nx; - columns(rowOffset - 10) = rowIdx - nx*ny - 1; - columns(rowOffset - 9) = rowIdx - nx*ny; - columns(rowOffset - 8) = rowIdx - nx*ny + nx - 1; - columns(rowOffset - 7) = rowIdx - nx*ny + nx; - columns(rowOffset - 6) = rowIdx - nx - 1; - columns(rowOffset - 5) = rowIdx - nx; - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + nx - 1; - columns(rowOffset - 1) = rowIdx + nx; - if(topBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 12) = -1.0; - values(rowOffset - 11) = -1.0; - values(rowOffset - 10) = -2.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = -1.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 8.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = 0.0; - } + // Fill column indices + columns(rowOffset - 4) = rowIdx - nx; + columns(rowOffset - 3) = rowIdx - 1; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (bottomBC == 1 || backBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 1.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 3.0; + values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const zEdgeFETag&, const size_type idx) const { - // Compute row index - ordinal_type k = idx; - ordinal_type rowIdx = (k + 1)*ny*nx; - - // Compute rowOffset - size_type rowOffset = k*numEntriesPerGridPlane - + numEntriesBottomPlane + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 12) = rowIdx - nx*ny; - columns(rowOffset - 11) = rowIdx - nx*ny + 1; - columns(rowOffset - 10) = rowIdx - nx*ny + nx; - columns(rowOffset - 9) = rowIdx - nx*ny + nx + 1; - columns(rowOffset - 8) = rowIdx; - columns(rowOffset - 7) = rowIdx + 1; - columns(rowOffset - 6) = rowIdx + nx; - columns(rowOffset - 5) = rowIdx + nx + 1; - columns(rowOffset - 4) = rowIdx + ny*nx; - columns(rowOffset - 3) = rowIdx + ny*nx + 1; - columns(rowOffset - 2) = rowIdx + ny*nx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx + nx + 1; - if(frontBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 1.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = -1.0; - values(rowOffset - 10) = -1.0; - values(rowOffset - 9) = -1.0; - values(rowOffset - 8) = 8.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = -2.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } - - // Compute row index - k = idx; - ordinal_type i = nx - 2; - rowIdx = (k + 1)*ny*nx + i + 1; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane + numEntriesBottomPlane - + i*faceStencilLength + 2*edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 12) = rowIdx - nx*ny - 1; - columns(rowOffset - 11) = rowIdx - nx*ny; - columns(rowOffset - 10) = rowIdx - nx*ny + nx - 1; - columns(rowOffset - 9) = rowIdx - nx*ny + nx; - columns(rowOffset - 8) = rowIdx - 1; - columns(rowOffset - 7) = rowIdx; - columns(rowOffset - 6) = rowIdx + nx - 1; - columns(rowOffset - 5) = rowIdx + nx; - columns(rowOffset - 4) = rowIdx + ny*nx - 1; - columns(rowOffset - 3) = rowIdx + ny*nx; - columns(rowOffset - 2) = rowIdx + ny*nx + nx - 1; - columns(rowOffset - 1) = rowIdx + ny*nx + nx; - if(frontBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 1.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 12) = -1.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = -1.0; - values(rowOffset - 9) = -1.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 8.0; - values(rowOffset - 6) = -2.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } + rowIdx = (nz - 1) * ny * nx; + rowOffset = (nz - 2) * numEntriesPerGridPlane + numEntriesBottomPlane + + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 4) = rowIdx - ny * nx; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + 1; + columns(rowOffset - 1) = rowIdx + nx; + if (topBC == 1 || frontBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 3.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } - // Compute row index - k = idx; - ordinal_type j = ny - 2; - rowIdx = (k + 1)*ny*nx - + (j + 1)*nx; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane - + numEntriesBottomPlane - + j*numEntriesPerGridRow + numEntriesFrontRow - + edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 12) = rowIdx - nx*ny - nx; - columns(rowOffset - 11) = rowIdx - nx*ny - nx + 1; - columns(rowOffset - 10) = rowIdx - nx*ny; - columns(rowOffset - 9) = rowIdx - nx*ny + 1; - columns(rowOffset - 8) = rowIdx - nx; - columns(rowOffset - 7) = rowIdx - nx + 1; - columns(rowOffset - 6) = rowIdx; - columns(rowOffset - 5) = rowIdx + 1; - columns(rowOffset - 4) = rowIdx + ny*nx - nx; - columns(rowOffset - 3) = rowIdx + ny*nx - nx + 1; - columns(rowOffset - 2) = rowIdx + ny*nx; - columns(rowOffset - 1) = rowIdx + ny*nx + 1; - if(backBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 1.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 12) = -1.0; - values(rowOffset - 11) = -1.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = -1.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = -2.0; - values(rowOffset - 6) = 8.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = -1.0; - } + rowIdx = (nz - 1) * ny * nx + nx - 1; + rowOffset = (nz - 2) * numEntriesPerGridPlane + numEntriesBottomPlane + + numEntriesBottomFrontRow; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 4) = rowIdx - ny * nx; + columns(rowOffset - 3) = rowIdx - 1; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + nx; + if (topBC == 1 || frontBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 1.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 3.0; + values(rowOffset - 1) = -1.0; + } - // Compute row index - k = idx; - j = ny - 2; - i = nx - 2; - rowIdx = (k + 1)*ny*nx + (j + 1)*nx + i + 1; - - // Compute rowOffset - rowOffset = k*numEntriesPerGridPlane - + numEntriesBottomPlane - + j*numEntriesPerGridRow + numEntriesFrontRow - + i*faceStencilLength + 2*edgeStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 12) = rowIdx - nx*ny - nx - 1; - columns(rowOffset - 11) = rowIdx - nx*ny - nx; - columns(rowOffset - 10) = rowIdx - nx*ny - 1; - columns(rowOffset - 9) = rowIdx - nx*ny; - columns(rowOffset - 8) = rowIdx - nx - 1; - columns(rowOffset - 7) = rowIdx - nx; - columns(rowOffset - 6) = rowIdx - 1; - columns(rowOffset - 5) = rowIdx; - columns(rowOffset - 4) = rowIdx + ny*nx - nx - 1; - columns(rowOffset - 3) = rowIdx + ny*nx - nx; - columns(rowOffset - 2) = rowIdx + ny*nx - 1; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(backBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 12) = 0.0; - values(rowOffset - 11) = 0.0; - values(rowOffset - 10) = 0.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 1.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 12) = -1.0; - values(rowOffset - 11) = -1.0; - values(rowOffset - 10) = -1.0; - values(rowOffset - 9) = 0.0; - values(rowOffset - 8) = -2.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 8.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = 0.0; - } + rowIdx = nz * ny * nx - nx; + rowOffset = (nz - 2) * numEntriesPerGridPlane + numEntriesBottomPlane + + (ny - 2) * numEntriesFrontRow + numEntriesBottomFrontRow + + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 4) = rowIdx - ny * nx; + columns(rowOffset - 3) = rowIdx - nx; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + 1; + if (topBC == 1 || backBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 1.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 3.0; + values(rowOffset - 1) = -1.0; } - KOKKOS_INLINE_FUNCTION - void operator() (const cornerFETag&, const size_type /*idx*/) const { - // Bottom corners - ordinal_type rowIdx = 0; - size_type rowOffset = cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 8) = rowIdx; - columns(rowOffset - 7) = rowIdx + 1; - columns(rowOffset - 6) = rowIdx + nx; - columns(rowOffset - 5) = rowIdx + nx + 1; - columns(rowOffset - 4) = rowIdx + ny*nx; - columns(rowOffset - 3) = rowIdx + ny*nx + 1; - columns(rowOffset - 2) = rowIdx + ny*nx + nx; - columns(rowOffset - 1) = rowIdx + ny*nx + nx + 1; - if(bottomBC == 1 || frontBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 8) = 1.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 8) = 4.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } + rowIdx = nz * ny * nx - 1; + rowOffset = numEntries; + rowmap(rowIdx + 1) = rowOffset; - rowIdx = nx - 1; - rowOffset = numEntriesBottomFrontRow; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 8) = rowIdx - 1; - columns(rowOffset - 7) = rowIdx; - columns(rowOffset - 6) = rowIdx + nx - 1; - columns(rowOffset - 5) = rowIdx + nx; - columns(rowOffset - 4) = rowIdx + ny*nx - 1; - columns(rowOffset - 3) = rowIdx + ny*nx; - columns(rowOffset - 2) = rowIdx + ny*nx + nx - 1; - columns(rowOffset - 1) = rowIdx + ny*nx + nx; - if(bottomBC == 1 || frontBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 1.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 4.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = -1.0; - } + // Fill column indices + columns(rowOffset - 4) = rowIdx - ny * nx; + columns(rowOffset - 3) = rowIdx - nx; + columns(rowOffset - 2) = rowIdx - 1; + columns(rowOffset - 1) = rowIdx; + if (topBC == 1 || backBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 1.0; + } else { + // Fill values + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = 3.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const interiorFETag&, const size_type idx) const { + // Compute row index + const ordinal_type k = idx / ((ny - 2) * (nx - 2)); + const ordinal_type rem = idx % ((ny - 2) * (nx - 2)); + const ordinal_type j = rem / (nx - 2); + const ordinal_type i = rem % (nx - 2); + const ordinal_type rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i + 1; + + // Compute rowOffset + const size_type rowOffset = + k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesPerGridRow + numEntriesFrontRow + + (i + 1) * interiorStencilLength + faceStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 27) = rowIdx - ny * nx - nx - 1; + columns(rowOffset - 26) = rowIdx - ny * nx - nx; + columns(rowOffset - 25) = rowIdx - ny * nx - nx + 1; + columns(rowOffset - 24) = rowIdx - ny * nx - 1; + columns(rowOffset - 23) = rowIdx - ny * nx; + columns(rowOffset - 22) = rowIdx - ny * nx + 1; + columns(rowOffset - 21) = rowIdx - ny * nx + nx - 1; + columns(rowOffset - 20) = rowIdx - ny * nx + nx; + columns(rowOffset - 19) = rowIdx - ny * nx + nx + 1; + columns(rowOffset - 18) = rowIdx - nx - 1; + columns(rowOffset - 17) = rowIdx - nx; + columns(rowOffset - 16) = rowIdx - nx + 1; + columns(rowOffset - 15) = rowIdx - 1; + columns(rowOffset - 14) = rowIdx; + columns(rowOffset - 13) = rowIdx + 1; + columns(rowOffset - 12) = rowIdx + nx - 1; + columns(rowOffset - 11) = rowIdx + nx; + columns(rowOffset - 10) = rowIdx + nx + 1; + columns(rowOffset - 9) = rowIdx + nx * ny - nx - 1; + columns(rowOffset - 8) = rowIdx + nx * ny - nx; + columns(rowOffset - 7) = rowIdx + nx * ny - nx + 1; + columns(rowOffset - 6) = rowIdx + nx * ny - 1; + columns(rowOffset - 5) = rowIdx + nx * ny; + columns(rowOffset - 4) = rowIdx + nx * ny + 1; + columns(rowOffset - 3) = rowIdx + nx * ny + nx - 1; + columns(rowOffset - 2) = rowIdx + nx * ny + nx; + columns(rowOffset - 1) = rowIdx + nx * ny + nx + 1; + + // Fill values + values(rowOffset - 27) = -1.0; + values(rowOffset - 26) = -2.0; + values(rowOffset - 25) = -1.0; + values(rowOffset - 24) = -2.0; + values(rowOffset - 23) = 0.0; + values(rowOffset - 22) = -2.0; + values(rowOffset - 21) = -1.0; + values(rowOffset - 20) = -2.0; + values(rowOffset - 19) = -1.0; + values(rowOffset - 18) = -2.0; + values(rowOffset - 17) = 0.0; + values(rowOffset - 16) = -2.0; + values(rowOffset - 15) = 0.0; + values(rowOffset - 14) = 32.0; + values(rowOffset - 13) = 0.0; + values(rowOffset - 12) = -2.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = -2.0; + values(rowOffset - 9) = -1.0; + values(rowOffset - 8) = -2.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = -2.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = -2.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -2.0; + values(rowOffset - 1) = -1.0; + } + + KOKKOS_INLINE_FUNCTION + void operator()(const xFaceFETag&, const size_type idx) const { + /*******************/ + /* x == 0 face */ + /*******************/ + // Compute row index + ordinal_type k = idx / (ny - 2); + ordinal_type j = idx % (ny - 2); + ordinal_type i = 0; + ordinal_type rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i; + + // Compute rowOffset + size_type rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesPerGridRow + numEntriesFrontRow + + faceStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 18) = rowIdx - ny * nx - nx; + columns(rowOffset - 17) = rowIdx - ny * nx - nx + 1; + columns(rowOffset - 16) = rowIdx - ny * nx; + columns(rowOffset - 15) = rowIdx - ny * nx + 1; + columns(rowOffset - 14) = rowIdx - ny * nx + nx; + columns(rowOffset - 13) = rowIdx - ny * nx + nx + 1; + columns(rowOffset - 12) = rowIdx - nx; + columns(rowOffset - 11) = rowIdx - nx + 1; + columns(rowOffset - 10) = rowIdx; + columns(rowOffset - 9) = rowIdx + 1; + columns(rowOffset - 8) = rowIdx + nx; + columns(rowOffset - 7) = rowIdx + nx + 1; + columns(rowOffset - 6) = rowIdx + nx * ny - nx; + columns(rowOffset - 5) = rowIdx + nx * ny - nx + 1; + columns(rowOffset - 4) = rowIdx + nx * ny; + columns(rowOffset - 3) = rowIdx + nx * ny + 1; + columns(rowOffset - 2) = rowIdx + nx * ny + nx; + columns(rowOffset - 1) = rowIdx + nx * ny + nx + 1; + if (leftBC == 1) { + // Fill values + values(rowOffset - 18) = 0.0; + values(rowOffset - 17) = 0.0; + values(rowOffset - 16) = 0.0; + values(rowOffset - 15) = 0.0; + values(rowOffset - 14) = 0.0; + values(rowOffset - 13) = 0.0; + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 1.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 18) = -1.0; + values(rowOffset - 17) = -1.0; + values(rowOffset - 16) = 0.0; + values(rowOffset - 15) = -2.0; + values(rowOffset - 14) = -1.0; + values(rowOffset - 13) = -1.0; + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = -2.0; + values(rowOffset - 10) = 16.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = -2.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = -2.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } - rowIdx = (ny - 1)*nx; - rowOffset = (ny - 2)*numEntriesFrontRow - + numEntriesBottomFrontRow + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 8) = rowIdx - nx; - columns(rowOffset - 7) = rowIdx - nx + 1; - columns(rowOffset - 6) = rowIdx; - columns(rowOffset - 5) = rowIdx + 1; - columns(rowOffset - 4) = rowIdx + ny*nx - nx; - columns(rowOffset - 3) = rowIdx + ny*nx - nx + 1; - columns(rowOffset - 2) = rowIdx + ny*nx; - columns(rowOffset - 1) = rowIdx + ny*nx + 1; - if(bottomBC == 1 || backBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 1.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = 4.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = -1.0; - } + /********************/ + /* x == nx face */ + /********************/ + // Compute row index + k = idx / (ny - 2); + j = idx % (ny - 2); + i = nx - 1; + rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + (j + 1) * numEntriesPerGridRow + numEntriesFrontRow; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 18) = rowIdx - ny * nx - nx - 1; + columns(rowOffset - 17) = rowIdx - ny * nx - nx; + columns(rowOffset - 16) = rowIdx - ny * nx - 1; + columns(rowOffset - 15) = rowIdx - ny * nx; + columns(rowOffset - 14) = rowIdx - ny * nx + nx - 1; + columns(rowOffset - 13) = rowIdx - ny * nx + nx; + columns(rowOffset - 12) = rowIdx - nx - 1; + columns(rowOffset - 11) = rowIdx - nx; + columns(rowOffset - 10) = rowIdx - 1; + columns(rowOffset - 9) = rowIdx; + columns(rowOffset - 8) = rowIdx + nx - 1; + columns(rowOffset - 7) = rowIdx + nx; + columns(rowOffset - 6) = rowIdx + nx * ny - nx - 1; + columns(rowOffset - 5) = rowIdx + nx * ny - nx; + columns(rowOffset - 4) = rowIdx + nx * ny - 1; + columns(rowOffset - 3) = rowIdx + nx * ny; + columns(rowOffset - 2) = rowIdx + nx * ny + nx - 1; + columns(rowOffset - 1) = rowIdx + nx * ny + nx; + if (rightBC == 1) { + // Fill values + values(rowOffset - 18) = 0.0; + values(rowOffset - 17) = 0.0; + values(rowOffset - 16) = 0.0; + values(rowOffset - 15) = 0.0; + values(rowOffset - 14) = 0.0; + values(rowOffset - 13) = 0.0; + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 1.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 18) = -1.0; + values(rowOffset - 17) = -1.0; + values(rowOffset - 16) = -2.0; + values(rowOffset - 15) = 0.0; + values(rowOffset - 14) = -1.0; + values(rowOffset - 13) = -1.0; + values(rowOffset - 12) = -2.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 16.0; + values(rowOffset - 8) = -2.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = -2.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const yFaceFETag&, const size_type idx) const { + /*******************/ + /* y == 0 face */ + /*******************/ + // Compute row index + ordinal_type k = idx / (nx - 2); + ordinal_type i = idx % (nx - 2); + ordinal_type rowIdx = (k + 1) * ny * nx + i + 1; + + // Compute rowOffset + size_type rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + (i + 1) * faceStencilLength + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 18) = rowIdx - ny * nx - 1; + columns(rowOffset - 17) = rowIdx - ny * nx; + columns(rowOffset - 16) = rowIdx - ny * nx + 1; + columns(rowOffset - 15) = rowIdx - ny * nx + nx - 1; + columns(rowOffset - 14) = rowIdx - ny * nx + nx; + columns(rowOffset - 13) = rowIdx - ny * nx + nx + 1; + columns(rowOffset - 12) = rowIdx - 1; + columns(rowOffset - 11) = rowIdx; + columns(rowOffset - 10) = rowIdx + 1; + columns(rowOffset - 9) = rowIdx + nx - 1; + columns(rowOffset - 8) = rowIdx + nx; + columns(rowOffset - 7) = rowIdx + nx + 1; + columns(rowOffset - 6) = rowIdx + nx * ny - 1; + columns(rowOffset - 5) = rowIdx + nx * ny; + columns(rowOffset - 4) = rowIdx + nx * ny + 1; + columns(rowOffset - 3) = rowIdx + nx * ny + nx - 1; + columns(rowOffset - 2) = rowIdx + nx * ny + nx; + columns(rowOffset - 1) = rowIdx + nx * ny + nx + 1; + if (frontBC == 1) { + // Fill values + values(rowOffset - 18) = 0.0; + values(rowOffset - 17) = 0.0; + values(rowOffset - 16) = 0.0; + values(rowOffset - 15) = 0.0; + values(rowOffset - 14) = 0.0; + values(rowOffset - 13) = 0.0; + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 1.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 18) = -1.0; + values(rowOffset - 17) = 0.0; + values(rowOffset - 16) = -1.0; + values(rowOffset - 15) = -1.0; + values(rowOffset - 14) = -2.0; + values(rowOffset - 13) = -1.0; + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 16.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = -2.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = -2.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -2.0; + values(rowOffset - 1) = -1.0; + } - rowIdx = ny*nx - 1; - rowOffset = numEntriesBottomPlane; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 8) = rowIdx - nx - 1; - columns(rowOffset - 7) = rowIdx - nx; - columns(rowOffset - 6) = rowIdx - 1; - columns(rowOffset - 5) = rowIdx; - columns(rowOffset - 4) = rowIdx + ny*nx - nx - 1; - columns(rowOffset - 3) = rowIdx + ny*nx - nx; - columns(rowOffset - 2) = rowIdx + ny*nx - 1; - columns(rowOffset - 1) = rowIdx + ny*nx; - if(bottomBC == 1 || backBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 1.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 8) = -1.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 4.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = 0.0; - } + /********************/ + /* y == ny face */ + /********************/ + // Compute row index + k = idx / (nx - 2); + ordinal_type j = ny - 2; + i = idx % (nx - 2); + rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i + 1; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesPerGridRow + numEntriesFrontRow + + (i + 1) * faceStencilLength + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 18) = rowIdx - ny * nx - nx - 1; + columns(rowOffset - 17) = rowIdx - ny * nx - nx; + columns(rowOffset - 16) = rowIdx - ny * nx - 1; + columns(rowOffset - 15) = rowIdx - ny * nx - 1; + columns(rowOffset - 14) = rowIdx - ny * nx; + columns(rowOffset - 13) = rowIdx - ny * nx + 1; + columns(rowOffset - 12) = rowIdx - nx - 1; + columns(rowOffset - 11) = rowIdx - nx; + columns(rowOffset - 10) = rowIdx - nx + 1; + columns(rowOffset - 9) = rowIdx - 1; + columns(rowOffset - 8) = rowIdx; + columns(rowOffset - 7) = rowIdx + 1; + columns(rowOffset - 6) = rowIdx + nx * ny - nx - 1; + columns(rowOffset - 5) = rowIdx + nx * ny - nx; + columns(rowOffset - 4) = rowIdx + nx * ny - nx + 1; + columns(rowOffset - 3) = rowIdx + nx * ny - 1; + columns(rowOffset - 2) = rowIdx + nx * ny; + columns(rowOffset - 1) = rowIdx + nx * ny + 1; + if (backBC == 1) { + // Fill values + values(rowOffset - 18) = 0.0; + values(rowOffset - 17) = 0.0; + values(rowOffset - 16) = 0.0; + values(rowOffset - 15) = 0.0; + values(rowOffset - 14) = 0.0; + values(rowOffset - 13) = 0.0; + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 1.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 18) = -1.0; + values(rowOffset - 17) = -2.0; + values(rowOffset - 16) = -1.0; + values(rowOffset - 15) = -1.0; + values(rowOffset - 14) = 0.0; + values(rowOffset - 13) = -1.0; + values(rowOffset - 12) = -2.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = -2.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 16.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -2.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = -1.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const zFaceFETag&, const size_type idx) const { + /*******************/ + /* z == 0 face */ + /*******************/ + // Compute row index + ordinal_type j = idx / (nx - 2); + ordinal_type i = idx % (nx - 2); + ordinal_type rowIdx = (j + 1) * nx + i + 1; + + // Compute rowOffset + size_type rowOffset = j * numEntriesFrontRow + numEntriesBottomFrontRow + + (i + 1) * faceStencilLength + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 18) = rowIdx - nx - 1; + columns(rowOffset - 17) = rowIdx - nx; + columns(rowOffset - 16) = rowIdx - nx + 1; + columns(rowOffset - 15) = rowIdx - 1; + columns(rowOffset - 14) = rowIdx; + columns(rowOffset - 13) = rowIdx + 1; + columns(rowOffset - 12) = rowIdx + nx - 1; + columns(rowOffset - 11) = rowIdx + nx; + columns(rowOffset - 10) = rowIdx + nx + 1; + columns(rowOffset - 9) = rowIdx + nx * ny - nx - 1; + columns(rowOffset - 8) = rowIdx + nx * ny - nx; + columns(rowOffset - 7) = rowIdx + nx * ny - nx + 1; + columns(rowOffset - 6) = rowIdx + nx * ny - 1; + columns(rowOffset - 5) = rowIdx + nx * ny; + columns(rowOffset - 4) = rowIdx + nx * ny + 1; + columns(rowOffset - 3) = rowIdx + nx * ny + nx - 1; + columns(rowOffset - 2) = rowIdx + nx * ny + nx; + columns(rowOffset - 1) = rowIdx + nx * ny + nx + 1; + if (bottomBC == 1) { + // Fill values + values(rowOffset - 18) = 0.0; + values(rowOffset - 17) = 0.0; + values(rowOffset - 16) = 0.0; + values(rowOffset - 15) = 0.0; + values(rowOffset - 14) = 1.0; + values(rowOffset - 13) = 0.0; + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 18) = -1.0; + values(rowOffset - 17) = 0.0; + values(rowOffset - 16) = -1.0; + values(rowOffset - 15) = 0.0; + values(rowOffset - 14) = 16.0; + values(rowOffset - 13) = 0.0; + values(rowOffset - 12) = -1.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = -1.0; + values(rowOffset - 9) = -1.0; + values(rowOffset - 8) = -2.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = -2.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = -2.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -2.0; + values(rowOffset - 1) = -1.0; + } - // Top corners - rowIdx = (nz - 1)*ny*nx; - rowOffset = (nz - 2)*numEntriesPerGridPlane - + numEntriesBottomPlane + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 8) = rowIdx - ny*nx; - columns(rowOffset - 7) = rowIdx - ny*nx + 1; - columns(rowOffset - 6) = rowIdx - ny*nx + nx; - columns(rowOffset - 5) = rowIdx - ny*nx + nx + 1; - columns(rowOffset - 4) = rowIdx; - columns(rowOffset - 3) = rowIdx + 1; - columns(rowOffset - 2) = rowIdx + nx; - columns(rowOffset - 1) = rowIdx + nx + 1; - if(topBC == 1 || frontBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 4.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = -1.0; - } + /********************/ + /* z == nz face */ + /********************/ + // Compute row index + ordinal_type k = nz - 2; + j = idx / (nx - 2); + i = idx % (nx - 2); + rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i + 1; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesFrontRow + numEntriesBottomFrontRow + + (i + 1) * faceStencilLength + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 18) = rowIdx - nx * ny - nx - 1; + columns(rowOffset - 17) = rowIdx - nx * ny - nx; + columns(rowOffset - 16) = rowIdx - nx * ny - nx + 1; + columns(rowOffset - 15) = rowIdx - nx * ny - 1; + columns(rowOffset - 14) = rowIdx - nx * ny; + columns(rowOffset - 13) = rowIdx - nx * ny + 1; + columns(rowOffset - 12) = rowIdx - nx * ny + nx - 1; + columns(rowOffset - 11) = rowIdx - nx * ny + nx; + columns(rowOffset - 10) = rowIdx - nx * ny + nx + 1; + columns(rowOffset - 9) = rowIdx - nx - 1; + columns(rowOffset - 8) = rowIdx - nx; + columns(rowOffset - 7) = rowIdx - nx + 1; + columns(rowOffset - 6) = rowIdx - 1; + columns(rowOffset - 5) = rowIdx; + columns(rowOffset - 4) = rowIdx + 1; + columns(rowOffset - 3) = rowIdx + nx - 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + nx + 1; + if (topBC == 1) { + // Fill values + values(rowOffset - 18) = 0.0; + values(rowOffset - 17) = 0.0; + values(rowOffset - 16) = 0.0; + values(rowOffset - 15) = 0.0; + values(rowOffset - 14) = 0.0; + values(rowOffset - 13) = 0.0; + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 1.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 18) = -1.0; + values(rowOffset - 17) = -2.0; + values(rowOffset - 16) = -1.0; + values(rowOffset - 15) = -2.0; + values(rowOffset - 14) = 0.0; + values(rowOffset - 13) = -2.0; + values(rowOffset - 12) = -1.0; + values(rowOffset - 11) = -2.0; + values(rowOffset - 10) = -1.0; + values(rowOffset - 9) = -1.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 16.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = -1.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const xEdgeFETag&, const size_type idx) const { + // Compute row index + ordinal_type i = idx; + ordinal_type rowIdx = i + 1; + + // Compute rowOffset + size_type rowOffset = (i + 1) * edgeStencilLength + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 12) = rowIdx - 1; + columns(rowOffset - 11) = rowIdx; + columns(rowOffset - 10) = rowIdx + 1; + columns(rowOffset - 9) = rowIdx + nx - 1; + columns(rowOffset - 8) = rowIdx + nx; + columns(rowOffset - 7) = rowIdx + nx + 1; + columns(rowOffset - 6) = rowIdx + nx * ny - 1; + columns(rowOffset - 5) = rowIdx + nx * ny; + columns(rowOffset - 4) = rowIdx + nx * ny + 1; + columns(rowOffset - 3) = rowIdx + nx * ny + nx - 1; + columns(rowOffset - 2) = rowIdx + nx * ny + nx; + columns(rowOffset - 1) = rowIdx + nx * ny + nx + 1; + if (bottomBC == 1 || frontBC == 1) { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 1.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 8.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = -1.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -2.0; + values(rowOffset - 1) = -1.0; + } - rowIdx = (nz - 1)*ny*nx + nx - 1; - rowOffset = (nz - 2)*numEntriesPerGridPlane - + numEntriesBottomPlane + numEntriesBottomFrontRow; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 8) = rowIdx - ny*nx - 1; - columns(rowOffset - 7) = rowIdx - ny*nx; - columns(rowOffset - 6) = rowIdx - ny*nx + nx - 1; - columns(rowOffset - 5) = rowIdx - ny*nx + nx; - columns(rowOffset - 4) = rowIdx - 1; - columns(rowOffset - 3) = rowIdx; - columns(rowOffset - 2) = rowIdx + nx - 1; - columns(rowOffset - 1) = rowIdx + nx; - if(topBC == 1 || frontBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 1.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 8) = -1.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 4.0; - values(rowOffset - 2) = -1.0; - values(rowOffset - 1) = 0.0; - } + // Compute row index + ordinal_type j = ny - 2; + i = idx; + rowIdx = (j + 1) * nx + i + 1; + + // Compute rowOffset + rowOffset = j * numEntriesFrontRow + numEntriesBottomFrontRow + + (i + 1) * edgeStencilLength + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 12) = rowIdx - nx - 1; + columns(rowOffset - 11) = rowIdx - nx; + columns(rowOffset - 10) = rowIdx - nx + 1; + columns(rowOffset - 9) = rowIdx - 1; + columns(rowOffset - 8) = rowIdx; + columns(rowOffset - 7) = rowIdx + 1; + columns(rowOffset - 6) = rowIdx + nx * ny - nx - 1; + columns(rowOffset - 5) = rowIdx + nx * ny - nx; + columns(rowOffset - 4) = rowIdx + nx * ny - nx + 1; + columns(rowOffset - 3) = rowIdx + nx * ny - 1; + columns(rowOffset - 2) = rowIdx + nx * ny; + columns(rowOffset - 1) = rowIdx + nx * ny + 1; + if (bottomBC == 1 || backBC == 1) { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 1.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 12) = -1.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = -1.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 8.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -2.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = -1.0; + } - rowIdx = nz*ny*nx - nx; - rowOffset = (nz - 2)*numEntriesPerGridPlane - + numEntriesBottomPlane + (ny - 2)*numEntriesFrontRow - + numEntriesBottomFrontRow + cornerStencilLength; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 8) = rowIdx - ny*nx - nx; - columns(rowOffset - 7) = rowIdx - ny*nx - nx + 1; - columns(rowOffset - 6) = rowIdx - ny*nx; - columns(rowOffset - 5) = rowIdx - ny*nx + 1; - columns(rowOffset - 4) = rowIdx - nx; - columns(rowOffset - 3) = rowIdx - nx + 1; - columns(rowOffset - 2) = rowIdx; - columns(rowOffset - 1) = rowIdx + 1; - if(topBC == 1 || backBC == 1 || leftBC == 1) { - // Fill values - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 1.0; - values(rowOffset - 1) = 0.0; - } else { - // Fill values - values(rowOffset - 8) = -1.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = -1.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = -1.0; - values(rowOffset - 2) = 4.0; - values(rowOffset - 1) = 0.0; - } + // Compute row index + ordinal_type k = nz - 2; + i = idx; + rowIdx = (k + 1) * ny * nx + i + 1; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + (i + 1) * edgeStencilLength + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 12) = rowIdx - nx * ny - 1; + columns(rowOffset - 11) = rowIdx - nx * ny; + columns(rowOffset - 10) = rowIdx - nx * ny + 1; + columns(rowOffset - 9) = rowIdx - nx * ny + nx - 1; + columns(rowOffset - 8) = rowIdx - nx * ny + nx; + columns(rowOffset - 7) = rowIdx - nx * ny + nx + 1; + columns(rowOffset - 6) = rowIdx - 1; + columns(rowOffset - 5) = rowIdx; + columns(rowOffset - 4) = rowIdx + 1; + columns(rowOffset - 3) = rowIdx + nx - 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + nx + 1; + if (topBC == 1 || frontBC == 1) { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 1.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 12) = -1.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = -1.0; + values(rowOffset - 9) = -1.0; + values(rowOffset - 8) = -2.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 8.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = -1.0; + } - rowIdx = nz*ny*nx - 1; - rowOffset = numEntries; - rowmap(rowIdx + 1) = rowOffset; - - // Fill column indices - columns(rowOffset - 8) = rowIdx - ny*nx - nx - 1; - columns(rowOffset - 7) = rowIdx - ny*nx - nx; - columns(rowOffset - 6) = rowIdx - ny*nx - 1; - columns(rowOffset - 5) = rowIdx - ny*nx; - columns(rowOffset - 4) = rowIdx - nx - 1; - columns(rowOffset - 3) = rowIdx - nx; - columns(rowOffset - 2) = rowIdx - 1; - columns(rowOffset - 1) = rowIdx; - if(topBC == 1 || backBC == 1 || rightBC == 1) { - // Fill values - values(rowOffset - 8) = 0.0; - values(rowOffset - 7) = 0.0; - values(rowOffset - 6) = 0.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = 0.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 1.0; - } else { - // Fill values - values(rowOffset - 8) = -1.0; - values(rowOffset - 7) = -1.0; - values(rowOffset - 6) = -1.0; - values(rowOffset - 5) = 0.0; - values(rowOffset - 4) = -1.0; - values(rowOffset - 3) = 0.0; - values(rowOffset - 2) = 0.0; - values(rowOffset - 1) = 4.0; - } + // Compute row index + k = nz - 2; + j = ny - 2; + i = idx; + rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i + 1; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesFrontRow + numEntriesBottomFrontRow + + (i + 1) * edgeStencilLength + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 12) = rowIdx - nx * ny - nx - 1; + columns(rowOffset - 11) = rowIdx - nx * ny - nx; + columns(rowOffset - 10) = rowIdx - nx * ny - nx + 1; + columns(rowOffset - 9) = rowIdx - nx * ny - 1; + columns(rowOffset - 8) = rowIdx - nx * ny; + columns(rowOffset - 7) = rowIdx - nx * ny + 1; + columns(rowOffset - 6) = rowIdx - nx - 1; + columns(rowOffset - 5) = rowIdx - nx; + columns(rowOffset - 4) = rowIdx - nx + 1; + columns(rowOffset - 3) = rowIdx - 1; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + 1; + if (topBC == 1 || backBC == 1) { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 1.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 12) = -1.0; + values(rowOffset - 11) = -2.0; + values(rowOffset - 10) = -1.0; + values(rowOffset - 9) = -1.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 8.0; + values(rowOffset - 1) = 0.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const yEdgeFETag&, const size_type idx) const { + // Compute row index + ordinal_type j = idx; + ordinal_type rowIdx = (j + 1) * nx; + + // Compute rowOffset + size_type rowOffset = + j * numEntriesFrontRow + numEntriesBottomFrontRow + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 12) = rowIdx - nx; + columns(rowOffset - 11) = rowIdx - nx + 1; + columns(rowOffset - 10) = rowIdx; + columns(rowOffset - 9) = rowIdx + 1; + columns(rowOffset - 8) = rowIdx + nx; + columns(rowOffset - 7) = rowIdx + nx + 1; + columns(rowOffset - 6) = rowIdx + nx * ny - nx; + columns(rowOffset - 5) = rowIdx + nx * ny - nx + 1; + columns(rowOffset - 4) = rowIdx + nx * ny; + columns(rowOffset - 3) = rowIdx + nx * ny + 1; + columns(rowOffset - 2) = rowIdx + nx * ny + nx; + columns(rowOffset - 1) = rowIdx + nx * ny + nx + 1; + if (bottomBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 1.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = -1.0; + values(rowOffset - 10) = 8.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = -2.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - }; - template - CrsMatrix_t generate_structured_matrix3D(const std::string stencil, - const mat_structure& structure) { + // Compute row index + j = idx; + ordinal_type i = nx - 1; + rowIdx = (j + 1) * nx + i; + + // Compute rowOffset + rowOffset = (j + 1) * numEntriesFrontRow + numEntriesBottomFrontRow; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 12) = rowIdx - nx - 1; + columns(rowOffset - 11) = rowIdx - nx; + columns(rowOffset - 10) = rowIdx - 1; + columns(rowOffset - 9) = rowIdx; + columns(rowOffset - 8) = rowIdx + nx - 1; + columns(rowOffset - 7) = rowIdx + nx; + columns(rowOffset - 6) = rowIdx + nx * ny - nx - 1; + columns(rowOffset - 5) = rowIdx + nx * ny - nx; + columns(rowOffset - 4) = rowIdx + nx * ny - 1; + columns(rowOffset - 3) = rowIdx + nx * ny; + columns(rowOffset - 2) = rowIdx + nx * ny + nx - 1; + columns(rowOffset - 1) = rowIdx + nx * ny + nx; + if (bottomBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 12) = -1.0; + values(rowOffset - 11) = -1.0; + values(rowOffset - 10) = -2.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = -1.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 8.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = 0.0; + } - typedef typename CrsMatrix_t::StaticCrsGraphType graph_t; - typedef typename CrsMatrix_t::row_map_type::non_const_type row_map_view_t; - typedef typename CrsMatrix_t::index_type::non_const_type cols_view_t; - typedef typename CrsMatrix_t::values_type::non_const_type scalar_view_t; - typedef typename CrsMatrix_t::non_const_size_type size_type; - typedef typename CrsMatrix_t::non_const_ordinal_type ordinal_type; + // Compute row index + ordinal_type k = nz - 2; + j = idx; + rowIdx = (k + 1) * ny * nx + (j + 1) * nx; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesFrontRow + numEntriesBottomFrontRow + + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 12) = rowIdx - nx * ny - 1; + columns(rowOffset - 11) = rowIdx - nx * ny; + columns(rowOffset - 10) = rowIdx - nx * ny + 1; + columns(rowOffset - 9) = rowIdx - nx * ny + nx - 1; + columns(rowOffset - 8) = rowIdx - nx * ny + nx; + columns(rowOffset - 7) = rowIdx - nx * ny + nx + 1; + columns(rowOffset - 6) = rowIdx - 1; + columns(rowOffset - 5) = rowIdx; + columns(rowOffset - 4) = rowIdx + 1; + columns(rowOffset - 3) = rowIdx + nx - 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + nx + 1; + if (topBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 1.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 12) = -1.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = -1.0; + values(rowOffset - 9) = -1.0; + values(rowOffset - 8) = -2.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 8.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = -1.0; + } - int stencil_type = 0; - if (stencil == "FD") { - stencil_type = FD; - } else if (stencil == "FE") { - stencil_type = FE; + // Compute row index + k = nz - 2; + j = idx; + i = nx - 1; + rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + (j + 1) * numEntriesFrontRow + numEntriesBottomFrontRow; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 12) = rowIdx - nx * ny - nx - 1; + columns(rowOffset - 11) = rowIdx - nx * ny - nx; + columns(rowOffset - 10) = rowIdx - nx * ny - 1; + columns(rowOffset - 9) = rowIdx - nx * ny; + columns(rowOffset - 8) = rowIdx - nx * ny + nx - 1; + columns(rowOffset - 7) = rowIdx - nx * ny + nx; + columns(rowOffset - 6) = rowIdx - nx - 1; + columns(rowOffset - 5) = rowIdx - nx; + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + nx - 1; + columns(rowOffset - 1) = rowIdx + nx; + if (topBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; } else { - std::ostringstream os; - os << "Test::generate_structured_matrix3D only accepts stencil: FD and FEM, you passed: " - << stencil <<" !" << std::endl; - Kokkos::Impl::throw_runtime_exception (os.str ()); - } - - // Extract geometric data - const ordinal_type nx = structure(0,0); - const ordinal_type ny = structure(1,0); - const ordinal_type nz = structure(2,0); - const ordinal_type numNodes = ny*nx*nz; - const ordinal_type leftBC = structure(0,1); - const ordinal_type rightBC = structure(0,2); - const ordinal_type frontBC = structure(1,1); - const ordinal_type backBC = structure(1,2); - const ordinal_type bottomBC = structure(2,1); - const ordinal_type topBC = structure(2,2); - const ordinal_type numInterior = (nx - leftBC - rightBC)*(ny - frontBC - backBC) - *(nz - bottomBC - topBC); - const ordinal_type numFace = - (leftBC + rightBC)*(ny - frontBC - backBC)*(nz - bottomBC - topBC) - + (frontBC + backBC)*(nx - leftBC - rightBC)*(nz - bottomBC - topBC) - + (bottomBC + topBC)*(nx - leftBC - rightBC)*(ny - frontBC - backBC); - const ordinal_type numEdge = - (frontBC*bottomBC + frontBC*topBC + backBC*bottomBC + backBC*topBC)*(nx - leftBC - rightBC) - + (leftBC*bottomBC + leftBC*topBC + rightBC*bottomBC + rightBC*topBC)*(ny - frontBC - backBC) - + (leftBC*frontBC + leftBC*backBC + rightBC*frontBC + rightBC*backBC)*(nz - bottomBC - topBC); - const ordinal_type numCorner = leftBC*frontBC*bottomBC + rightBC*frontBC*bottomBC - + leftBC*backBC*bottomBC + rightBC*backBC*bottomBC - + leftBC*frontBC*topBC + rightBC*frontBC*topBC - + leftBC*backBC*topBC + rightBC*backBC*topBC; - ordinal_type interiorStencilLength = 0, faceStencilLength = 0, edgeStencilLength = 0, cornerStencilLength = 0; - - if(stencil_type == FD) { - interiorStencilLength = 7; - faceStencilLength = 6; - edgeStencilLength = 5; - cornerStencilLength = 4; - } else if(stencil_type == FE) { - interiorStencilLength = 27; - faceStencilLength = 18; - edgeStencilLength = 12; - cornerStencilLength = 8; + // Fill values + values(rowOffset - 12) = -1.0; + values(rowOffset - 11) = -1.0; + values(rowOffset - 10) = -2.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = -1.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 8.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = 0.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const zEdgeFETag&, const size_type idx) const { + // Compute row index + ordinal_type k = idx; + ordinal_type rowIdx = (k + 1) * ny * nx; + + // Compute rowOffset + size_type rowOffset = + k * numEntriesPerGridPlane + numEntriesBottomPlane + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 12) = rowIdx - nx * ny; + columns(rowOffset - 11) = rowIdx - nx * ny + 1; + columns(rowOffset - 10) = rowIdx - nx * ny + nx; + columns(rowOffset - 9) = rowIdx - nx * ny + nx + 1; + columns(rowOffset - 8) = rowIdx; + columns(rowOffset - 7) = rowIdx + 1; + columns(rowOffset - 6) = rowIdx + nx; + columns(rowOffset - 5) = rowIdx + nx + 1; + columns(rowOffset - 4) = rowIdx + ny * nx; + columns(rowOffset - 3) = rowIdx + ny * nx + 1; + columns(rowOffset - 2) = rowIdx + ny * nx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx + nx + 1; + if (frontBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 1.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = -1.0; + values(rowOffset - 10) = -1.0; + values(rowOffset - 9) = -1.0; + values(rowOffset - 8) = 8.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = -2.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; } - const size_type numEntries = numInterior*interiorStencilLength - + numFace*faceStencilLength - + numEdge*edgeStencilLength - + numCorner*cornerStencilLength; + // Compute row index + k = idx; + ordinal_type i = nx - 2; + rowIdx = (k + 1) * ny * nx + i + 1; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + i * faceStencilLength + 2 * edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 12) = rowIdx - nx * ny - 1; + columns(rowOffset - 11) = rowIdx - nx * ny; + columns(rowOffset - 10) = rowIdx - nx * ny + nx - 1; + columns(rowOffset - 9) = rowIdx - nx * ny + nx; + columns(rowOffset - 8) = rowIdx - 1; + columns(rowOffset - 7) = rowIdx; + columns(rowOffset - 6) = rowIdx + nx - 1; + columns(rowOffset - 5) = rowIdx + nx; + columns(rowOffset - 4) = rowIdx + ny * nx - 1; + columns(rowOffset - 3) = rowIdx + ny * nx; + columns(rowOffset - 2) = rowIdx + ny * nx + nx - 1; + columns(rowOffset - 1) = rowIdx + ny * nx + nx; + if (frontBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 1.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 12) = -1.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = -1.0; + values(rowOffset - 9) = -1.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 8.0; + values(rowOffset - 6) = -2.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } - // Create matrix data - row_map_view_t rowmap_view ("rowmap_view", numNodes + 1); - cols_view_t columns_view("colsmap_view", numEntries); - scalar_view_t values_view ("values_view", numEntries); + // Compute row index + k = idx; + ordinal_type j = ny - 2; + rowIdx = (k + 1) * ny * nx + (j + 1) * nx; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesPerGridRow + numEntriesFrontRow + + edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 12) = rowIdx - nx * ny - nx; + columns(rowOffset - 11) = rowIdx - nx * ny - nx + 1; + columns(rowOffset - 10) = rowIdx - nx * ny; + columns(rowOffset - 9) = rowIdx - nx * ny + 1; + columns(rowOffset - 8) = rowIdx - nx; + columns(rowOffset - 7) = rowIdx - nx + 1; + columns(rowOffset - 6) = rowIdx; + columns(rowOffset - 5) = rowIdx + 1; + columns(rowOffset - 4) = rowIdx + ny * nx - nx; + columns(rowOffset - 3) = rowIdx + ny * nx - nx + 1; + columns(rowOffset - 2) = rowIdx + ny * nx; + columns(rowOffset - 1) = rowIdx + ny * nx + 1; + if (backBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 1.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 12) = -1.0; + values(rowOffset - 11) = -1.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = -1.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = -2.0; + values(rowOffset - 6) = 8.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = -1.0; + } - // Fill the CrsGraph and the CrsMatrix - // To start simple we construct 2D 5pt stencil Laplacian. - // We assume Neumann boundary conditions on the edge of the domain. + // Compute row index + k = idx; + j = ny - 2; + i = nx - 2; + rowIdx = (k + 1) * ny * nx + (j + 1) * nx + i + 1; + + // Compute rowOffset + rowOffset = k * numEntriesPerGridPlane + numEntriesBottomPlane + + j * numEntriesPerGridRow + numEntriesFrontRow + + i * faceStencilLength + 2 * edgeStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 12) = rowIdx - nx * ny - nx - 1; + columns(rowOffset - 11) = rowIdx - nx * ny - nx; + columns(rowOffset - 10) = rowIdx - nx * ny - 1; + columns(rowOffset - 9) = rowIdx - nx * ny; + columns(rowOffset - 8) = rowIdx - nx - 1; + columns(rowOffset - 7) = rowIdx - nx; + columns(rowOffset - 6) = rowIdx - 1; + columns(rowOffset - 5) = rowIdx; + columns(rowOffset - 4) = rowIdx + ny * nx - nx - 1; + columns(rowOffset - 3) = rowIdx + ny * nx - nx; + columns(rowOffset - 2) = rowIdx + ny * nx - 1; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (backBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 12) = 0.0; + values(rowOffset - 11) = 0.0; + values(rowOffset - 10) = 0.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 1.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 12) = -1.0; + values(rowOffset - 11) = -1.0; + values(rowOffset - 10) = -1.0; + values(rowOffset - 9) = 0.0; + values(rowOffset - 8) = -2.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 8.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = 0.0; + } + } + + KOKKOS_INLINE_FUNCTION + void operator()(const cornerFETag&, const size_type /*idx*/) const { + // Bottom corners + ordinal_type rowIdx = 0; + size_type rowOffset = cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 8) = rowIdx; + columns(rowOffset - 7) = rowIdx + 1; + columns(rowOffset - 6) = rowIdx + nx; + columns(rowOffset - 5) = rowIdx + nx + 1; + columns(rowOffset - 4) = rowIdx + ny * nx; + columns(rowOffset - 3) = rowIdx + ny * nx + 1; + columns(rowOffset - 2) = rowIdx + ny * nx + nx; + columns(rowOffset - 1) = rowIdx + ny * nx + nx + 1; + if (bottomBC == 1 || frontBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 8) = 1.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 8) = 4.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } - fill_3D_matrix_functor fill_3D_matrix(stencil_type, nx, ny, nz, - leftBC, rightBC, frontBC, - backBC, bottomBC, topBC, - rowmap_view, columns_view, - values_view); + rowIdx = nx - 1; + rowOffset = numEntriesBottomFrontRow; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 8) = rowIdx - 1; + columns(rowOffset - 7) = rowIdx; + columns(rowOffset - 6) = rowIdx + nx - 1; + columns(rowOffset - 5) = rowIdx + nx; + columns(rowOffset - 4) = rowIdx + ny * nx - 1; + columns(rowOffset - 3) = rowIdx + ny * nx; + columns(rowOffset - 2) = rowIdx + ny * nx + nx - 1; + columns(rowOffset - 1) = rowIdx + ny * nx + nx; + if (bottomBC == 1 || frontBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 1.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 4.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = -1.0; + } - fill_3D_matrix.compute(); + rowIdx = (ny - 1) * nx; + rowOffset = (ny - 2) * numEntriesFrontRow + numEntriesBottomFrontRow + + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 8) = rowIdx - nx; + columns(rowOffset - 7) = rowIdx - nx + 1; + columns(rowOffset - 6) = rowIdx; + columns(rowOffset - 5) = rowIdx + 1; + columns(rowOffset - 4) = rowIdx + ny * nx - nx; + columns(rowOffset - 3) = rowIdx + ny * nx - nx + 1; + columns(rowOffset - 2) = rowIdx + ny * nx; + columns(rowOffset - 1) = rowIdx + ny * nx + 1; + if (bottomBC == 1 || backBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 1.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = 4.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = -1.0; + } - graph_t static_graph (columns_view, rowmap_view); - std::string name; - if(stencil_type == FD) { - name = "CrsMatrixFD"; - } else if(stencil_type == FE) { - name = "CrsMatrixFE"; + rowIdx = ny * nx - 1; + rowOffset = numEntriesBottomPlane; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 8) = rowIdx - nx - 1; + columns(rowOffset - 7) = rowIdx - nx; + columns(rowOffset - 6) = rowIdx - 1; + columns(rowOffset - 5) = rowIdx; + columns(rowOffset - 4) = rowIdx + ny * nx - nx - 1; + columns(rowOffset - 3) = rowIdx + ny * nx - nx; + columns(rowOffset - 2) = rowIdx + ny * nx - 1; + columns(rowOffset - 1) = rowIdx + ny * nx; + if (bottomBC == 1 || backBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 1.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 8) = -1.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 4.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = 0.0; } - return CrsMatrix_t(name, numNodes, values_view, static_graph); + // Top corners + rowIdx = (nz - 1) * ny * nx; + rowOffset = (nz - 2) * numEntriesPerGridPlane + numEntriesBottomPlane + + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 8) = rowIdx - ny * nx; + columns(rowOffset - 7) = rowIdx - ny * nx + 1; + columns(rowOffset - 6) = rowIdx - ny * nx + nx; + columns(rowOffset - 5) = rowIdx - ny * nx + nx + 1; + columns(rowOffset - 4) = rowIdx; + columns(rowOffset - 3) = rowIdx + 1; + columns(rowOffset - 2) = rowIdx + nx; + columns(rowOffset - 1) = rowIdx + nx + 1; + if (topBC == 1 || frontBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = 4.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = -1.0; + } - } // generate_structured_matrix3D + rowIdx = (nz - 1) * ny * nx + nx - 1; + rowOffset = (nz - 2) * numEntriesPerGridPlane + numEntriesBottomPlane + + numEntriesBottomFrontRow; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 8) = rowIdx - ny * nx - 1; + columns(rowOffset - 7) = rowIdx - ny * nx; + columns(rowOffset - 6) = rowIdx - ny * nx + nx - 1; + columns(rowOffset - 5) = rowIdx - ny * nx + nx; + columns(rowOffset - 4) = rowIdx - 1; + columns(rowOffset - 3) = rowIdx; + columns(rowOffset - 2) = rowIdx + nx - 1; + columns(rowOffset - 1) = rowIdx + nx; + if (topBC == 1 || frontBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 1.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 8) = -1.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 4.0; + values(rowOffset - 2) = -1.0; + values(rowOffset - 1) = 0.0; + } -} + rowIdx = nz * ny * nx - nx; + rowOffset = (nz - 2) * numEntriesPerGridPlane + numEntriesBottomPlane + + (ny - 2) * numEntriesFrontRow + numEntriesBottomFrontRow + + cornerStencilLength; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 8) = rowIdx - ny * nx - nx; + columns(rowOffset - 7) = rowIdx - ny * nx - nx + 1; + columns(rowOffset - 6) = rowIdx - ny * nx; + columns(rowOffset - 5) = rowIdx - ny * nx + 1; + columns(rowOffset - 4) = rowIdx - nx; + columns(rowOffset - 3) = rowIdx - nx + 1; + columns(rowOffset - 2) = rowIdx; + columns(rowOffset - 1) = rowIdx + 1; + if (topBC == 1 || backBC == 1 || leftBC == 1) { + // Fill values + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 1.0; + values(rowOffset - 1) = 0.0; + } else { + // Fill values + values(rowOffset - 8) = -1.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = -1.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = -1.0; + values(rowOffset - 2) = 4.0; + values(rowOffset - 1) = 0.0; + } -#endif // KOKKOSKERNELS_TEST_STRUCTURE_MATRIX_HPP + rowIdx = nz * ny * nx - 1; + rowOffset = numEntries; + rowmap(rowIdx + 1) = rowOffset; + + // Fill column indices + columns(rowOffset - 8) = rowIdx - ny * nx - nx - 1; + columns(rowOffset - 7) = rowIdx - ny * nx - nx; + columns(rowOffset - 6) = rowIdx - ny * nx - 1; + columns(rowOffset - 5) = rowIdx - ny * nx; + columns(rowOffset - 4) = rowIdx - nx - 1; + columns(rowOffset - 3) = rowIdx - nx; + columns(rowOffset - 2) = rowIdx - 1; + columns(rowOffset - 1) = rowIdx; + if (topBC == 1 || backBC == 1 || rightBC == 1) { + // Fill values + values(rowOffset - 8) = 0.0; + values(rowOffset - 7) = 0.0; + values(rowOffset - 6) = 0.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = 0.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 1.0; + } else { + // Fill values + values(rowOffset - 8) = -1.0; + values(rowOffset - 7) = -1.0; + values(rowOffset - 6) = -1.0; + values(rowOffset - 5) = 0.0; + values(rowOffset - 4) = -1.0; + values(rowOffset - 3) = 0.0; + values(rowOffset - 2) = 0.0; + values(rowOffset - 1) = 4.0; + } + } +}; + +template +CrsMatrix_t generate_structured_matrix3D(const std::string stencil, + const mat_structure& structure) { + typedef typename CrsMatrix_t::StaticCrsGraphType graph_t; + typedef typename CrsMatrix_t::row_map_type::non_const_type row_map_view_t; + typedef typename CrsMatrix_t::index_type::non_const_type cols_view_t; + typedef typename CrsMatrix_t::values_type::non_const_type scalar_view_t; + typedef typename CrsMatrix_t::non_const_size_type size_type; + typedef typename CrsMatrix_t::non_const_ordinal_type ordinal_type; + + int stencil_type = 0; + if (stencil == "FD") { + stencil_type = FD; + } else if (stencil == "FE") { + stencil_type = FE; + } else { + std::ostringstream os; + os << "Test::generate_structured_matrix3D only accepts stencil: FD and " + "FEM, you passed: " + << stencil << " !" << std::endl; + KokkosKernels::Impl::throw_runtime_exception(os.str()); + } + + // Extract geometric data + const ordinal_type nx = structure(0, 0); + const ordinal_type ny = structure(1, 0); + const ordinal_type nz = structure(2, 0); + const ordinal_type numNodes = ny * nx * nz; + const ordinal_type leftBC = structure(0, 1); + const ordinal_type rightBC = structure(0, 2); + const ordinal_type frontBC = structure(1, 1); + const ordinal_type backBC = structure(1, 2); + const ordinal_type bottomBC = structure(2, 1); + const ordinal_type topBC = structure(2, 2); + const ordinal_type numInterior = (nx - leftBC - rightBC) * + (ny - frontBC - backBC) * + (nz - bottomBC - topBC); + const ordinal_type numFace = + (leftBC + rightBC) * (ny - frontBC - backBC) * (nz - bottomBC - topBC) + + (frontBC + backBC) * (nx - leftBC - rightBC) * (nz - bottomBC - topBC) + + (bottomBC + topBC) * (nx - leftBC - rightBC) * (ny - frontBC - backBC); + const ordinal_type numEdge = (frontBC * bottomBC + frontBC * topBC + + backBC * bottomBC + backBC * topBC) * + (nx - leftBC - rightBC) + + (leftBC * bottomBC + leftBC * topBC + + rightBC * bottomBC + rightBC * topBC) * + (ny - frontBC - backBC) + + (leftBC * frontBC + leftBC * backBC + + rightBC * frontBC + rightBC * backBC) * + (nz - bottomBC - topBC); + const ordinal_type numCorner = + leftBC * frontBC * bottomBC + rightBC * frontBC * bottomBC + + leftBC * backBC * bottomBC + rightBC * backBC * bottomBC + + leftBC * frontBC * topBC + rightBC * frontBC * topBC + + leftBC * backBC * topBC + rightBC * backBC * topBC; + ordinal_type interiorStencilLength = 0, faceStencilLength = 0, + edgeStencilLength = 0, cornerStencilLength = 0; + + if (stencil_type == FD) { + interiorStencilLength = 7; + faceStencilLength = 6; + edgeStencilLength = 5; + cornerStencilLength = 4; + } else if (stencil_type == FE) { + interiorStencilLength = 27; + faceStencilLength = 18; + edgeStencilLength = 12; + cornerStencilLength = 8; + } + + const size_type numEntries = + numInterior * interiorStencilLength + numFace * faceStencilLength + + numEdge * edgeStencilLength + numCorner * cornerStencilLength; + + // Create matrix data + row_map_view_t rowmap_view("rowmap_view", numNodes + 1); + cols_view_t columns_view("colsmap_view", numEntries); + scalar_view_t values_view("values_view", numEntries); + + // Fill the CrsGraph and the CrsMatrix + // To start simple we construct 2D 5pt stencil Laplacian. + // We assume Neumann boundary conditions on the edge of the domain. + + fill_3D_matrix_functor fill_3D_matrix( + stencil_type, nx, ny, nz, leftBC, rightBC, frontBC, backBC, bottomBC, + topBC, rowmap_view, columns_view, values_view); + + fill_3D_matrix.compute(); + + graph_t static_graph(columns_view, rowmap_view); + std::string name; + if (stencil_type == FD) { + name = "CrsMatrixFD"; + } else if (stencil_type == FE) { + name = "CrsMatrixFE"; + } + + return CrsMatrix_t(name, numNodes, values_view, static_graph); + +} // generate_structured_matrix3D + +} // namespace Test + +#endif // KOKKOSKERNELS_TEST_STRUCTURE_MATRIX_HPP diff --git a/unit_test/common/Test_Common.hpp b/unit_test/common/Test_Common.hpp index 038ed21fda..0a194071a8 100644 --- a/unit_test/common/Test_Common.hpp +++ b/unit_test/common/Test_Common.hpp @@ -10,5 +10,6 @@ #include #include #include +#include #endif // TEST_COMMON_HPP diff --git a/unit_test/common/Test_Common_Error.hpp b/unit_test/common/Test_Common_Error.hpp new file mode 100644 index 0000000000..1b30e8375f --- /dev/null +++ b/unit_test/common/Test_Common_Error.hpp @@ -0,0 +1,61 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Siva Rajamanickam (srajama@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef TEST_COMMON_ERROR_HPP +#define TEST_COMMON_ERROR_HPP + +#include "KokkosKernels_Error.hpp" + +void test_kokkoskernels_throw() { + const std::string my_throw_msg = + "Testing Kokkos Kernels' throw_runtime_exception."; + try { + KokkosKernels::Impl::throw_runtime_exception(my_throw_msg); + } catch (const std::runtime_error& e) { + } +} + +TEST_F(TestCategory, common_throw) { test_kokkoskernels_throw(); } + +#endif // TEST_COMMON_ERROR_HPP diff --git a/unit_test/sparse/Test_Sparse_spmv.hpp b/unit_test/sparse/Test_Sparse_spmv.hpp index 9bb41d5123..267e1ecfb5 100644 --- a/unit_test/sparse/Test_Sparse_spmv.hpp +++ b/unit_test/sparse/Test_Sparse_spmv.hpp @@ -10,6 +10,7 @@ #include "KokkosKernels_Controls.hpp" #include "KokkosKernels_default_types.hpp" +#include "KokkosKernels_Error.hpp" #include "Cuda/Kokkos_Cuda_Half.hpp" // #ifndef kokkos_complex_double @@ -1067,10 +1068,10 @@ Matrix expand_matrix(std::vector pattern, const int m, const int k, // check rows and columns for (const Coordinate &c : pattern) { if (c.i >= m) { - Kokkos::Impl::throw_runtime_exception("i exceeded matrix rows"); + KokkosKernels::Impl::throw_runtime_exception("i exceeded matrix rows"); } if (c.j >= k) { - Kokkos::Impl::throw_runtime_exception("j exceeded matrix cols"); + KokkosKernels::Impl::throw_runtime_exception("j exceeded matrix cols"); } } @@ -1146,10 +1147,10 @@ Matrix expand_matrix(std::vector pattern, const int m, const int k, // check rows and columns for (const Coordinate &c : pattern) { if (c.i >= m) { - Kokkos::Impl::throw_runtime_exception("i exceeded matrix rows"); + KokkosKernels::Impl::throw_runtime_exception("i exceeded matrix rows"); } if (c.j >= k) { - Kokkos::Impl::throw_runtime_exception("j exceeded matrix cols"); + KokkosKernels::Impl::throw_runtime_exception("j exceeded matrix cols"); } }