diff --git a/src/impl/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp b/src/impl/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp index a1ae213ea9..a6eec44449 100644 --- a/src/impl/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp +++ b/src/impl/tpls/KokkosSparse_spmv_bsrmatrix_tpl_spec_decl.hpp @@ -503,7 +503,7 @@ void spmv_block_impl_cusparse( default: { std::cerr << "Mode " << mode << " invalid for cusparse[*]bsrmv.\n"; throw std::invalid_argument("Invalid mode"); - } break; + } } #if (9000 <= CUDA_VERSION) @@ -599,7 +599,7 @@ void spm_mv_block_impl_cusparse( default: { std::cerr << "Mode " << mode << " invalid for cusparse[*]bsrmv.\n"; throw std::invalid_argument("Invalid mode"); - } break; + } } int colx = static_cast(x.extent(1)); diff --git a/src/sparse/KokkosSparse_spmv.hpp b/src/sparse/KokkosSparse_spmv.hpp index 8615145a6d..a08fced7da 100644 --- a/src/sparse/KokkosSparse_spmv.hpp +++ b/src/sparse/KokkosSparse_spmv.hpp @@ -1007,13 +1007,13 @@ void spmv(KokkosKernels::Experimental::Controls controls, const char mode[], return spmv(controls, mode, alpha, A_i, x_0, beta, y_0, RANK_ONE()); } // - return Experimental::Impl::SPMV_MV_BLOCKCRSMATRIX< - typename AMatrix_Internal::value_type, - typename AMatrix_Internal::ordinal_type, + Experimental::Impl::SPMV_MV_BLOCKCRSMATRIX< + typename AMatrix_Internal::const_value_type, + typename AMatrix_Internal::const_ordinal_type, typename AMatrix_Internal::device_type, typename AMatrix_Internal::memory_traits, - typename AMatrix_Internal::size_type, - typename XVector_Internal::value_type**, + typename AMatrix_Internal::const_size_type, + typename XVector_Internal::const_value_type**, typename XVector_Internal::array_layout, typename XVector_Internal::device_type, typename XVector_Internal::memory_traits, diff --git a/src/sparse/impl/KokkosSparse_spmv_blockcrsmatrix_spec.hpp b/src/sparse/impl/KokkosSparse_spmv_blockcrsmatrix_spec.hpp index 7132ec0fe1..14b75f1c39 100644 --- a/src/sparse/impl/KokkosSparse_spmv_blockcrsmatrix_spec.hpp +++ b/src/sparse/impl/KokkosSparse_spmv_blockcrsmatrix_spec.hpp @@ -101,10 +101,10 @@ struct spmv_mv_blockcrsmatrix_eti_spec_avail { const SCALAR_TYPE, const ORDINAL_TYPE, \ Kokkos::Device, \ Kokkos::MemoryTraits, const OFFSET_TYPE, \ - SCALAR_TYPE const *, LAYOUT_TYPE, \ + SCALAR_TYPE const **, LAYOUT_TYPE, \ Kokkos::Device, \ Kokkos::MemoryTraits, \ - SCALAR_TYPE *, LAYOUT_TYPE, \ + SCALAR_TYPE **, LAYOUT_TYPE, \ Kokkos::Device, \ Kokkos::MemoryTraits > { \ enum : bool { value = true }; \ diff --git a/unit_test/sparse/Test_Sparse.hpp b/unit_test/sparse/Test_Sparse.hpp index 2afa0fb2db..30639512c5 100644 --- a/unit_test/sparse/Test_Sparse.hpp +++ b/unit_test/sparse/Test_Sparse.hpp @@ -14,8 +14,8 @@ #include "Test_Sparse_spgemm.hpp" #include "Test_Sparse_spiluk.hpp" #include "Test_Sparse_spmv.hpp" -//#include "Test_Sparse_spmv_blockcrs.hpp" -//#include "Test_Sparse_spmv_bsr.hpp" +#include "Test_Sparse_spmv_blockcrs.hpp" +#include "Test_Sparse_spmv_bsr.hpp" #include "Test_Sparse_sptrsv.hpp" #include "Test_Sparse_trsv.hpp" diff --git a/unit_test/sparse/Test_Sparse_spmv_blockcrs.hpp b/unit_test/sparse/Test_Sparse_spmv_blockcrs.hpp index f570a2d5df..9ba09e353c 100644 --- a/unit_test/sparse/Test_Sparse_spmv_blockcrs.hpp +++ b/unit_test/sparse/Test_Sparse_spmv_blockcrs.hpp @@ -42,6 +42,7 @@ //@HEADER */ +#include #include #include #include @@ -128,36 +129,44 @@ void check_blockcrs_times_v(const char fOp[], scalar_t alpha, scalar_t beta, size_type nnz = static_cast(blockSize) * static_cast(blockSize) * mat_b1.nnz(); - // Fill block with random values - std::vector mat_val(nnz); - for (size_type ii = 0; ii < nnz; ++ii) set_random_value(mat_val[ii]); - // // Create graph for CrsMatrix // - std::vector mat_rowmap(nRow + 1, 0); - std::vector mat_colidx(nnz, 0); + Kokkos::View d_rowmap("crsmatrix", nRow + 1); + auto h_rowmap = Kokkos::create_mirror_view(d_rowmap); + + Kokkos::View d_colidx("crsmatrix", nnz); + auto h_colidx = Kokkos::create_mirror_view(d_colidx); + + Kokkos::View d_matval("crsmatrix", nnz); + auto h_matval = Kokkos::create_mirror_view(d_matval); + + for (size_type ii = 0; ii < nnz; ++ii) set_random_value(h_matval[ii]); for (lno_t ir = 0; ir < mat_b1.numRows(); ++ir) { - const auto jbeg = mat_b1.graph.row_map(ir); - const auto jend = mat_b1.graph.row_map(ir + 1); + const size_type jbeg = mat_b1.graph.row_map(ir); + const size_type jend = mat_b1.graph.row_map(ir + 1); for (lno_t ib = 0; ib < blockSize; ++ib) { - const lno_t my_row = ir * blockSize + ib; - mat_rowmap[my_row + 1] = mat_rowmap[my_row] + (jend - jbeg) * blockSize; - for (lno_t ijk = jbeg; ijk < jend; ++ijk) { + const lno_t my_row = ir * blockSize + ib; + h_rowmap[my_row + 1] = h_rowmap[my_row] + (jend - jbeg) * blockSize; + for (size_type ijk = jbeg; ijk < jend; ++ijk) { const auto col0 = mat_b1.graph.entries(ijk); for (lno_t jb = 0; jb < blockSize; ++jb) { - mat_colidx[mat_rowmap[my_row] + (ijk - jbeg) * blockSize + jb] = + h_colidx[h_rowmap[my_row] + (ijk - jbeg) * blockSize + jb] = col0 * blockSize + jb; } } } } // for (lno_t ir = 0; ir < mat_b1.numRows(); ++ir) + Kokkos::deep_copy(d_matval, h_matval); + Kokkos::deep_copy(d_colidx, h_colidx); + Kokkos::deep_copy(d_rowmap, h_rowmap); + // Create the CrsMatrix for the reference computation - crsMat_t Acrs("new_crs_matr", nRow, nCol, nnz, &mat_val[0], &mat_rowmap[0], - &mat_colidx[0]); + crsMat_t Acrs("new_crs_matr", nRow, nCol, nnz, d_matval, d_rowmap, + d_colidx); x_vector_type xref("new_right_hand_side", nRow); auto h_xref = Kokkos::create_mirror_view(xref); @@ -179,7 +188,7 @@ void check_blockcrs_times_v(const char fOp[], scalar_t alpha, scalar_t beta, // Compute the reference product KokkosSparse::spmv(fOp, alpha, Acrs, xref, beta, ycrs); - y_vector_type ybcrs("bsr_product_result", nRow); + y_vector_type ybcrs("bcrs_product_result", nRow); auto h_ybcrs = Kokkos::create_mirror_view(ybcrs); for (lno_t ir = 0; ir < nRow; ++ir) h_ybcrs(ir) = h_y0(ir); Kokkos::deep_copy(ybcrs, h_ybcrs); @@ -187,26 +196,27 @@ void check_blockcrs_times_v(const char fOp[], scalar_t alpha, scalar_t beta, // Create the BlockCrsMatrix KokkosSparse::Experimental::BlockCrsMatrix - Absr(Acrs, blockSize); + Abcrs(Acrs, blockSize); // Compute the product with the BlockCrsMatrix format - KokkosSparse::spmv(fOp, alpha, Absr, xref, beta, ybcrs); + KokkosSparse::spmv(fOp, alpha, Abcrs, xref, beta, ybcrs); // Compare the two products - double error = 0.0, maxNorm = 0.0; + using KATS = Kokkos::ArithTraits; + using mag_type = typename KATS::mag_type; + + const mag_type zero_mag = Kokkos::ArithTraits::zero(); + mag_type error = zero_mag, maxNorm = zero_mag; + Kokkos::deep_copy(h_ycrs, ycrs); Kokkos::deep_copy(h_ybcrs, ybcrs); for (lno_t ir = 0; ir < nRow; ++ir) { - error = std::max( - error, Kokkos::ArithTraits::abs(h_ycrs(ir) - h_ybcrs(ir))); - maxNorm = - std::max(maxNorm, Kokkos::ArithTraits::abs(h_ycrs(ir))); + error = std::max(error, KATS::abs(h_ycrs(ir) - h_ybcrs(ir))); + maxNorm = std::max(maxNorm, KATS::abs(h_ycrs(ir))); } - double tmps = - static_cast(Kokkos::ArithTraits::abs(alpha)) + - static_cast(Kokkos::ArithTraits::abs(beta)); - if ((tmps > 0.0) && (maxNorm == 0)) { + mag_type tmps = KATS::abs(alpha) + KATS::abs(beta); + if ((tmps > zero_mag) && (maxNorm == zero_mag)) { std::cout << " BlockCRSMatrix - SpMV times V >> blockSize " << blockSize << " maxNorm " << maxNorm << " error " << error << " alpha " << alpha << " beta " << beta << "\n"; @@ -216,9 +226,8 @@ void check_blockcrs_times_v(const char fOp[], scalar_t alpha, scalar_t beta, // // --- Factor ((nnz / nRow) + 1) = Average number of non-zeros per row // - const auto tol = ((nnz / nRow) + 1) * - static_cast(Kokkos::ArithTraits::abs( - Kokkos::ArithTraits::epsilon())); + const mag_type tol = ((static_cast(nnz) / nRow) + 1) * + Kokkos::ArithTraits::epsilon(); if (error > tol * maxNorm) { std::cout << " BlockCRSMatrix - SpMV times V >> blockSize " << blockSize << " ratio " << error / maxNorm << " tol " << tol << " maxNorm " @@ -231,7 +240,7 @@ void check_blockcrs_times_v(const char fOp[], scalar_t alpha, scalar_t beta, /// \brief Driver routine for checking BlockCrsMatrix times multiple vector template + typename layout, typename device> void check_blockcrs_times_mv(const char fOp[], scalar_t alpha, scalar_t beta, const lno_t bMax, int &num_errors) { // The mat_structure view is used to generate a matrix using @@ -255,7 +264,7 @@ void check_blockcrs_times_mv(const char fOp[], scalar_t alpha, scalar_t beta, typedef typename KokkosSparse::CrsMatrix crsMat_t; - typedef Kokkos::View block_vector_t; + typedef Kokkos::View block_vector_t; h_crsMat_t mat_b1 = Test::generate_structured_matrix3D("FD", mat_structure); @@ -273,41 +282,40 @@ void check_blockcrs_times_mv(const char fOp[], scalar_t alpha, scalar_t beta, size_type nnz = static_cast(blockSize) * static_cast(blockSize) * mat_b1.nnz(); - std::vector mat_val(nnz); - for (size_type ii = 0; ii < nnz; ++ii) set_random_value(mat_val[ii]); + Kokkos::View d_rowmap("crsmatrix", nRow + 1); + auto h_rowmap = Kokkos::create_mirror_view(d_rowmap); - // - // Create graph for CrsMatrix - // + Kokkos::View d_colidx("crsmatrix", nnz); + auto h_colidx = Kokkos::create_mirror_view(d_colidx); - std::vector mat_rowmap(nRow + 1); - std::vector mat_colidx(nnz); + Kokkos::View d_matval("crsmatrix", nnz); + auto h_matval = Kokkos::create_mirror_view(d_matval); - mat_rowmap.resize(nRow + 1); - auto *rowmap = &mat_rowmap[0]; - rowmap[0] = 0; - - mat_colidx.resize(nnz); - auto *cols = &mat_colidx[0]; + for (size_type ii = 0; ii < nnz; ++ii) set_random_value(h_matval[ii]); for (lno_t ir = 0; ir < mat_b1.numRows(); ++ir) { - const auto jbeg = mat_b1.graph.row_map(ir); - const auto jend = mat_b1.graph.row_map(ir + 1); + const size_type jbeg = mat_b1.graph.row_map(ir); + const size_type jend = mat_b1.graph.row_map(ir + 1); for (lno_t ib = 0; ib < blockSize; ++ib) { - const lno_t my_row = ir * blockSize + ib; - rowmap[my_row + 1] = rowmap[my_row] + (jend - jbeg) * blockSize; - for (lno_t ijk = jbeg; ijk < jend; ++ijk) { + const lno_t my_row = ir * blockSize + ib; + h_rowmap[my_row + 1] = h_rowmap[my_row] + (jend - jbeg) * blockSize; + for (size_type ijk = jbeg; ijk < jend; ++ijk) { const auto col0 = mat_b1.graph.entries(ijk); for (lno_t jb = 0; jb < blockSize; ++jb) { - cols[rowmap[my_row] + (ijk - jbeg) * blockSize + jb] = + h_colidx[h_rowmap[my_row] + (ijk - jbeg) * blockSize + jb] = col0 * blockSize + jb; } } } } // for (lno_t ir = 0; ir < mat_b1.numRows(); ++ir) + Kokkos::deep_copy(d_matval, h_matval); + Kokkos::deep_copy(d_colidx, h_colidx); + Kokkos::deep_copy(d_rowmap, h_rowmap); + // Create the CrsMatrix for the reference computation - crsMat_t Acrs("new_crs_matr", nRow, nCol, nnz, &mat_val[0], rowmap, cols); + crsMat_t Acrs("new_crs_matr", nRow, nCol, nnz, d_matval, d_rowmap, + d_colidx); block_vector_t xref("new_right_hand_side", nRow, nrhs); auto h_xref = Kokkos::create_mirror_view(xref); @@ -329,7 +337,7 @@ void check_blockcrs_times_mv(const char fOp[], scalar_t alpha, scalar_t beta, KokkosSparse::spmv(fOp, alpha, Acrs, xref, beta, ycrs); - block_vector_t ybcrs("bsr_product_result", nRow, nrhs); + block_vector_t ybcrs("bcrs_product_result", nRow, nrhs); auto h_ybcrs = Kokkos::create_mirror_view(ybcrs); for (int jc = 0; jc < nrhs; ++jc) for (lno_t ir = 0; ir < nRow; ++ir) h_ybcrs(ir, jc) = h_y0(ir, jc); @@ -338,38 +346,40 @@ void check_blockcrs_times_mv(const char fOp[], scalar_t alpha, scalar_t beta, // Create the BlockCrsMatrix KokkosSparse::Experimental::BlockCrsMatrix - Absr(Acrs, blockSize); + Abcrs(Acrs, blockSize); // Compute the product for the BlockCrsMatrix format - KokkosSparse::spmv(fOp, alpha, Absr, xref, beta, ybcrs); + KokkosSparse::spmv(fOp, alpha, Abcrs, xref, beta, ybcrs); Kokkos::deep_copy(h_ycrs, ycrs); Kokkos::deep_copy(h_ybcrs, ybcrs); // Compare the two products - double error = 0.0, maxNorm = 0.0; + using KATS = Kokkos::ArithTraits; + using mag_type = typename KATS::mag_type; + + const mag_type zero_mag = Kokkos::ArithTraits::zero(); + mag_type error = zero_mag, maxNorm = zero_mag; + for (int jc = 0; jc < nrhs; ++jc) { for (int ir = 0; ir < nRow; ++ir) { - error = std::max(error, Kokkos::ArithTraits::abs( - h_ycrs(ir, jc) - h_ybcrs(ir, jc))); - maxNorm = std::max(maxNorm, - Kokkos::ArithTraits::abs(h_ycrs(ir, jc))); + error = std::max(error, + KATS::abs(h_ycrs(ir, jc) - h_ybcrs(ir, jc))); + maxNorm = std::max(maxNorm, KATS::abs(h_ycrs(ir, jc))); } } - auto tol = ((nnz / nRow) + 1) * - static_cast(Kokkos::ArithTraits::abs( - Kokkos::ArithTraits::epsilon())); - - double tmps = - static_cast(Kokkos::ArithTraits::abs(alpha)) + - static_cast(Kokkos::ArithTraits::abs(beta)); - if ((tmps > 0.0) && (maxNorm == 0)) { + + const mag_type tmps = KATS::abs(alpha) + KATS::abs(beta); + if ((tmps > zero_mag) && (maxNorm == zero_mag)) { std::cout << " BlockCRSMatrix - SpMV times MV >> blockSize " << blockSize << " maxNorm " << maxNorm << " error " << error << " alpha " << alpha << " beta " << beta << "\n"; num_errors += 1; } + const mag_type tol = ((static_cast(nnz) / nRow) + 1) * + Kokkos::ArithTraits::epsilon(); + if (error > tol * maxNorm) { std::cout << " BlockCRSMatrix - SpMV times MV >> blockSize " << blockSize << " ratio " << error / maxNorm << " tol " << tol << " maxNorm " @@ -425,7 +435,7 @@ void testSpMVBlockCrsMatrix() { } template + typename layout, typename device> void testBlockCrsMatrix_SpM_MV() { // // Test for the operation Y <- alpha * Op(A) * X + beta * Y @@ -452,7 +462,7 @@ void testBlockCrsMatrix_SpM_MV() { auto alpha_s = static_cast(testAlphaBeta[ii]); auto beta_s = static_cast(testAlphaBeta[ii + 1]); num_errors = 0; - Test_BlockCrs::check_blockcrs_times_mv(&mode, alpha_s, beta_s, bMax, num_errors); if (num_errors > 0) { @@ -610,11 +620,12 @@ EXECUTE_BCRS_TIMES_VEC_TEST(kokkos_complex_float, int64_t, size_t, ////////////////////////// -#define EXECUTE_BCRS_TIMES_MVEC_TEST(SCALAR, ORDINAL, OFFSET, DEVICE) \ - TEST_F( \ - TestCategory, \ - sparse##_##bcrs_times_multivec##_##SCALAR##_##ORDINAL##_##OFFSET##_##DEVICE) { \ - testBlockCrsMatrix_SpM_MV(); \ +#define EXECUTE_BCRS_TIMES_MVEC_TEST(SCALAR, ORDINAL, OFFSET, LAYOUT, DEVICE) \ + TEST_F( \ + TestCategory, \ + sparse##_##bcrs_times_multivec##_##SCALAR##_##ORDINAL##_##OFFSET##_##LAYOUT##_##DEVICE) { \ + testBlockCrsMatrix_SpM_MV(); \ } #if (defined(KOKKOSKERNELS_INST_DOUBLE) && \ @@ -622,7 +633,12 @@ EXECUTE_BCRS_TIMES_VEC_TEST(kokkos_complex_float, int64_t, size_t, defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(double, int, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(double, int, int, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(double, int, int, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_DOUBLE) && \ @@ -630,7 +646,12 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(double, int, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(double, int64_t, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(double, int64_t, int, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(double, int64_t, int, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_DOUBLE) && \ @@ -638,7 +659,12 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(double, int64_t, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(double, int, size_t, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(double, int, size_t, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(double, int, size_t, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_DOUBLE) && \ @@ -646,7 +672,13 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(double, int, size_t, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(double, int64_t, size_t, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(double, int64_t, size_t, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(double, int64_t, size_t, LayoutRight, + TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_FLOAT) && \ @@ -654,7 +686,12 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(double, int64_t, size_t, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(float, int, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(float, int, int, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(float, int, int, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_FLOAT) && \ @@ -662,7 +699,12 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(float, int, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(float, int64_t, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(float, int64_t, int, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(float, int64_t, int, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_FLOAT) && \ @@ -670,7 +712,12 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(float, int64_t, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(float, int, size_t, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(float, int, size_t, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(float, int, size_t, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_FLOAT) && \ @@ -678,7 +725,12 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(float, int, size_t, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(float, int64_t, size_t, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(float, int64_t, size_t, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(float, int64_t, size_t, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_DOUBLE_) && \ @@ -686,7 +738,14 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(float, int64_t, size_t, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int, int, LayoutLeft, + TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int, int, LayoutRight, + TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_DOUBLE_) && \ @@ -694,7 +753,14 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, int, LayoutLeft, + TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, int, LayoutRight, + TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_DOUBLE_) && \ @@ -702,7 +768,14 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int, size_t, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int, size_t, LayoutLeft, + TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int, size_t, LayoutRight, + TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_DOUBLE_) && \ @@ -710,16 +783,29 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int, size_t, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, size_t, +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, size_t, LayoutLeft, TestExecSpace) #endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, size_t, + LayoutRight, TestExecSpace) +#endif +#endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_FLOAT_) && \ defined(KOKKOSKERNELS_INST_ORDINAL_INT) && \ defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int, int, LayoutLeft, + TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int, int, LayoutRight, + TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_FLOAT_) && \ @@ -727,7 +813,14 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, int, LayoutLeft, + TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, int, LayoutRight, + TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_FLOAT_) && \ @@ -735,7 +828,14 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int, size_t, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int, size_t, LayoutLeft, + TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int, size_t, LayoutRight, + TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_FLOAT_) && \ @@ -743,8 +843,14 @@ EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int, size_t, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, size_t, +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, size_t, LayoutLeft, + TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BCRS_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, size_t, LayoutRight, TestExecSpace) #endif +#endif #undef EXECUTE_BCRS_TIMES_MVEC_TEST diff --git a/unit_test/sparse/Test_Sparse_spmv_bsr.hpp b/unit_test/sparse/Test_Sparse_spmv_bsr.hpp index e6d3b65ac5..6ee68ca3c5 100644 --- a/unit_test/sparse/Test_Sparse_spmv_bsr.hpp +++ b/unit_test/sparse/Test_Sparse_spmv_bsr.hpp @@ -42,6 +42,7 @@ //@HEADER */ +#include #include #include #include @@ -96,33 +97,29 @@ inline void set_random_value(std::complex &v) { /// \param mat_rowmap[out] CRS-style row map for the block matrix /// \param mat_colidx[out] CRS-style column entries for the block matrix /// \param mat_val[out] Numerical (random) values -template +template void make_block_entries( const KokkosSparse::CrsMatrix &mat_b1, - int blockSize, std::vector &mat_rowmap, - std::vector &mat_colidx, std::vector &mat_val) { - lno_t nRow = blockSize * mat_b1.numRows(); + int blockSize, rowmap_type &mat_rowmap, colidx_type &mat_colidx, + values_type &mat_val) { size_t nnz = static_cast(blockSize) * static_cast(blockSize) * mat_b1.nnz(); - mat_val.resize(nnz); for (size_t ii = 0; ii < nnz; ++ii) set_random_value(mat_val[ii]); // // Create graph for CrsMatrix // - mat_rowmap.assign(nRow + 1, 0); - mat_colidx.assign(nnz, 0); - for (lno_t ir = 0; ir < mat_b1.numRows(); ++ir) { - const auto jbeg = mat_b1.graph.row_map(ir); - const auto jend = mat_b1.graph.row_map(ir + 1); + const size_type jbeg = mat_b1.graph.row_map(ir); + const size_type jend = mat_b1.graph.row_map(ir + 1); for (lno_t ib = 0; ib < blockSize; ++ib) { const lno_t my_row = ir * blockSize + ib; mat_rowmap[my_row + 1] = mat_rowmap[my_row] + (jend - jbeg) * blockSize; - for (auto ijk = jbeg; ijk < jend; ++ijk) { + for (size_type ijk = jbeg; ijk < jend; ++ijk) { const auto col0 = mat_b1.graph.entries(ijk); for (lno_t jb = 0; jb < blockSize; ++jb) { mat_colidx[mat_rowmap[my_row] + (ijk - jbeg) * blockSize + jb] = @@ -177,17 +174,26 @@ void check_bsrm_times_v(const char fOp[], scalar_t alpha, scalar_t beta, size_type nnz = static_cast(blockSize) * static_cast(blockSize) * mat_b1.nnz(); - std::vector mat_rowmap(nRow + 1, 0); - std::vector mat_colidx(nnz, 0); - std::vector mat_val(nnz); + Kokkos::View d_rowmap("crsmatrix", nRow + 1); + auto h_rowmap = Kokkos::create_mirror_view(d_rowmap); + + Kokkos::View d_colidx("crsmatrix", nnz); + auto h_colidx = Kokkos::create_mirror_view(d_colidx); + + Kokkos::View d_matval("crsmatrix", nnz); + auto h_matval = Kokkos::create_mirror_view(d_matval); // Create the entries - make_block_entries(mat_b1, blockSize, mat_rowmap, - mat_colidx, mat_val); + make_block_entries(mat_b1, blockSize, h_rowmap, + h_colidx, h_matval); + + Kokkos::deep_copy(d_matval, h_matval); + Kokkos::deep_copy(d_colidx, h_colidx); + Kokkos::deep_copy(d_rowmap, h_rowmap); // Create the CrsMatrix for the reference computation - crsMat_t Acrs("new_crs_matr", nRow, nCol, nnz, &mat_val[0], &mat_rowmap[0], - &mat_colidx[0]); + crsMat_t Acrs("new_crs_matr", nRow, nCol, nnz, d_matval, d_rowmap, + d_colidx); x_vector_type xref("new_right_hand_side", nRow); auto h_xref = Kokkos::create_mirror_view(xref); @@ -229,20 +235,21 @@ void check_bsrm_times_v(const char fOp[], scalar_t alpha, scalar_t beta, // // Compare the two products // - double error = 0.0, maxNorm = 0.0; + using KATS = Kokkos::ArithTraits; + using mag_type = typename KATS::mag_type; + + const mag_type zero_mag = Kokkos::ArithTraits::zero(); + mag_type error = zero_mag, maxNorm = zero_mag; + Kokkos::deep_copy(h_ycrs, ycrs); Kokkos::deep_copy(h_ybsr, ybsr); for (lno_t ir = 0; ir < nRow; ++ir) { - error = std::max( - error, Kokkos::ArithTraits::abs(h_ycrs(ir) - h_ybsr(ir))); - maxNorm = - std::max(maxNorm, Kokkos::ArithTraits::abs(h_ycrs(ir))); + error = std::max(error, KATS::abs(h_ycrs(ir) - h_ybsr(ir))); + maxNorm = std::max(maxNorm, KATS::abs(h_ycrs(ir))); } - double tmps = - static_cast(Kokkos::ArithTraits::abs(alpha)) + - static_cast(Kokkos::ArithTraits::abs(beta)); - if ((tmps > 0.0) && (maxNorm == 0)) { + mag_type tmps = KATS::abs(alpha) + KATS::abs(beta); + if ((tmps > zero_mag) && (maxNorm == zero_mag)) { std::cout << " BSR - SpMV times MV >> blockSize " << blockSize << " maxNorm " << maxNorm << " error " << error << " alpha " << alpha << " beta " << beta << "\n"; @@ -252,9 +259,8 @@ void check_bsrm_times_v(const char fOp[], scalar_t alpha, scalar_t beta, // // --- Factor ((nnz / nRow) + 1) = Average number of non-zeros per row // - const auto tol = ((nnz / nRow) + 1) * - static_cast(Kokkos::ArithTraits::abs( - Kokkos::ArithTraits::epsilon())); + const mag_type tol = ((static_cast(nnz) / nRow) + 1) * + Kokkos::ArithTraits::epsilon(); if (error > tol * maxNorm) { std::cout << " BSR - SpMV times V >> blockSize " << blockSize << " ratio " << error / maxNorm << " tol " << tol << " maxNorm " << maxNorm @@ -267,7 +273,7 @@ void check_bsrm_times_v(const char fOp[], scalar_t alpha, scalar_t beta, /// \brief Driver routine for checking BsrMatrix times multiple vector template + typename layout, typename device> void check_bsrm_times_mv(const char fOp[], scalar_t alpha, scalar_t beta, const lno_t bMax, int &num_errors) { // The mat_structure view is used to generate a matrix using @@ -291,7 +297,7 @@ void check_bsrm_times_mv(const char fOp[], scalar_t alpha, scalar_t beta, typedef typename KokkosSparse::CrsMatrix crsMat_t; - typedef Kokkos::View block_vector_t; + typedef Kokkos::View block_vector_t; h_crsMat_t mat_b1 = Test::generate_structured_matrix3D("FD", mat_structure); @@ -309,17 +315,26 @@ void check_bsrm_times_mv(const char fOp[], scalar_t alpha, scalar_t beta, size_type nnz = static_cast(blockSize) * static_cast(blockSize) * mat_b1.nnz(); - std::vector mat_rowmap(nRow + 1, 0); - std::vector mat_colidx(nnz, 0); - std::vector mat_val(nnz); + Kokkos::View d_rowmap("crsmatrix", nRow + 1); + auto h_rowmap = Kokkos::create_mirror_view(d_rowmap); + + Kokkos::View d_colidx("crsmatrix", nnz); + auto h_colidx = Kokkos::create_mirror_view(d_colidx); + + Kokkos::View d_matval("crsmatrix", nnz); + auto h_matval = Kokkos::create_mirror_view(d_matval); // Create the entries - make_block_entries(mat_b1, static_cast(blockSize), - mat_rowmap, mat_colidx, mat_val); + make_block_entries(mat_b1, blockSize, h_rowmap, + h_colidx, h_matval); + + Kokkos::deep_copy(d_matval, h_matval); + Kokkos::deep_copy(d_colidx, h_colidx); + Kokkos::deep_copy(d_rowmap, h_rowmap); // Create the CrsMatrix for the reference computation - crsMat_t Acrs("new_crs_matr", nRow, nCol, nnz, &mat_val[0], &mat_rowmap[0], - &mat_colidx[0]); + crsMat_t Acrs("new_crs_matr", nRow, nCol, nnz, d_matval, d_rowmap, + d_colidx); block_vector_t xref("new_right_hand_side", nRow, nrhs); auto h_xref = Kokkos::create_mirror_view(xref); @@ -366,29 +381,29 @@ void check_bsrm_times_mv(const char fOp[], scalar_t alpha, scalar_t beta, // // Compare the two products // - double error = 0.0, maxNorm = 0.0; + using KATS = Kokkos::ArithTraits; + using mag_type = typename KATS::mag_type; + + const mag_type zero_mag = Kokkos::ArithTraits::zero(); + mag_type error = zero_mag, maxNorm = zero_mag; for (int jc = 0; jc < nrhs; ++jc) { for (int ir = 0; ir < nRow; ++ir) { - error = std::max(error, Kokkos::ArithTraits::abs( - h_ycrs(ir, jc) - h_ybsr(ir, jc))); - maxNorm = std::max(maxNorm, - Kokkos::ArithTraits::abs(h_ycrs(ir, jc))); + error = std::max(error, + KATS::abs(h_ycrs(ir, jc) - h_ybsr(ir, jc))); + maxNorm = std::max(maxNorm, KATS::abs(h_ycrs(ir, jc))); } } - double tmps = - static_cast(Kokkos::ArithTraits::abs(alpha)) + - static_cast(Kokkos::ArithTraits::abs(beta)); - if ((tmps > 0.0) && (maxNorm == 0)) { + mag_type tmps = KATS::abs(alpha) + KATS::abs(beta); + if ((tmps > zero_mag) && (maxNorm == zero_mag)) { std::cout << " BSR - SpMV times MV >> blockSize " << blockSize << " maxNorm " << maxNorm << " error " << error << " alpha " << alpha << " beta " << beta << "\n"; num_errors += 1; } - auto tol = ((nnz / nRow) + 1) * - static_cast(Kokkos::ArithTraits::abs( - Kokkos::ArithTraits::epsilon())); + const mag_type tol = ((static_cast(nnz) / nRow) + 1) * + Kokkos::ArithTraits::epsilon(); if (error > tol * maxNorm) { std::cout << " BSR - SpMV times MV >> blockSize " << blockSize << " ratio " << error / maxNorm << " tol " << tol << " maxNorm " @@ -531,7 +546,7 @@ void testSpMVBsrMatrix() { } template + typename layout, typename device> void testBsrMatrix_SpM_MV() { // // Test for the operation Y <- alpha * Op(A) * X + beta * Y @@ -558,7 +573,7 @@ void testBsrMatrix_SpM_MV() { auto alpha_s = static_cast(testAlphaBeta[ii]); auto beta_s = static_cast(testAlphaBeta[ii + 1]); num_errors = 0; - Test_Bsr::check_bsrm_times_mv( + Test_Bsr::check_bsrm_times_mv( &mode, alpha_s, beta_s, bMax, num_errors); if (num_errors > 0) { printf( @@ -714,11 +729,11 @@ EXECUTE_BSR_TIMES_VEC_TEST(kokkos_complex_float, int64_t, size_t, TestExecSpace) ////////////////////////// -#define EXECUTE_BSR_TIMES_MVEC_TEST(SCALAR, ORDINAL, OFFSET, DEVICE) \ - TEST_F( \ - TestCategory, \ - sparse##_##bsrmat_times_multivec##_##SCALAR##_##ORDINAL##_##OFFSET##_##DEVICE) { \ - testBsrMatrix_SpM_MV(); \ +#define EXECUTE_BSR_TIMES_MVEC_TEST(SCALAR, ORDINAL, OFFSET, LAYOUT, DEVICE) \ + TEST_F( \ + TestCategory, \ + sparse##_##bsrmat_times_multivec##_##SCALAR##_##ORDINAL##_##OFFSET##_##LAYOUT##_##DEVICE) { \ + testBsrMatrix_SpM_MV(); \ } #if (defined(KOKKOSKERNELS_INST_DOUBLE) && \ @@ -726,7 +741,12 @@ EXECUTE_BSR_TIMES_VEC_TEST(kokkos_complex_float, int64_t, size_t, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(double, int, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(double, int, int, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(double, int, int, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_DOUBLE) && \ @@ -734,7 +754,12 @@ EXECUTE_BSR_TIMES_MVEC_TEST(double, int, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(double, int64_t, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(double, int64_t, int, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(double, int64_t, int, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_DOUBLE) && \ @@ -742,7 +767,12 @@ EXECUTE_BSR_TIMES_MVEC_TEST(double, int64_t, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(double, int, size_t, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(double, int, size_t, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(double, int, size_t, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_DOUBLE) && \ @@ -750,7 +780,12 @@ EXECUTE_BSR_TIMES_MVEC_TEST(double, int, size_t, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(double, int64_t, size_t, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(double, int64_t, size_t, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(double, int64_t, size_t, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_FLOAT) && \ @@ -758,7 +793,12 @@ EXECUTE_BSR_TIMES_MVEC_TEST(double, int64_t, size_t, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(float, int, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(float, int, int, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(float, int, int, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_FLOAT) && \ @@ -766,7 +806,12 @@ EXECUTE_BSR_TIMES_MVEC_TEST(float, int, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(float, int64_t, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(float, int64_t, int, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(float, int64_t, int, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_FLOAT) && \ @@ -774,7 +819,12 @@ EXECUTE_BSR_TIMES_MVEC_TEST(float, int64_t, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(float, int, size_t, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(float, int, size_t, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(float, int, size_t, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_FLOAT) && \ @@ -782,7 +832,12 @@ EXECUTE_BSR_TIMES_MVEC_TEST(float, int, size_t, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(float, int64_t, size_t, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(float, int64_t, size_t, LayoutLeft, TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(float, int64_t, size_t, LayoutRight, TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_DOUBLE_) && \ @@ -790,7 +845,14 @@ EXECUTE_BSR_TIMES_MVEC_TEST(float, int64_t, size_t, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int, int, LayoutLeft, + TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int, int, LayoutRight, + TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_DOUBLE_) && \ @@ -798,7 +860,14 @@ EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, int, LayoutLeft, + TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, int, LayoutRight, + TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_DOUBLE_) && \ @@ -806,7 +875,14 @@ EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int, size_t, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int, size_t, LayoutLeft, + TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int, size_t, LayoutRight, + TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_DOUBLE_) && \ @@ -814,16 +890,29 @@ EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int, size_t, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, size_t, +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, size_t, LayoutLeft, TestExecSpace) #endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_double, int64_t, size_t, LayoutRight, + TestExecSpace) +#endif +#endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_FLOAT_) && \ defined(KOKKOSKERNELS_INST_ORDINAL_INT) && \ defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int, int, LayoutLeft, + TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int, int, LayoutRight, + TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_FLOAT_) && \ @@ -831,7 +920,14 @@ EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_INT)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, int, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, int, LayoutLeft, + TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, int, LayoutRight, + TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_FLOAT_) && \ @@ -839,7 +935,14 @@ EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, int, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int, size_t, TestExecSpace) +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int, size_t, LayoutLeft, + TestExecSpace) +#endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int, size_t, LayoutRight, + TestExecSpace) +#endif #endif #if (defined(KOKKOSKERNELS_INST_KOKKOS_COMPLEX_FLOAT_) && \ @@ -847,8 +950,14 @@ EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int, size_t, TestExecSpace) defined(KOKKOSKERNELS_INST_OFFSET_SIZE_T)) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) -EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, size_t, +#if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, size_t, LayoutLeft, TestExecSpace) #endif +#if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) +EXECUTE_BSR_TIMES_MVEC_TEST(kokkos_complex_float, int64_t, size_t, LayoutRight, + TestExecSpace) +#endif +#endif #undef EXECUTE_BSR_TIMES_MVEC_TEST