diff --git a/CMakeLists.txt b/CMakeLists.txt index 893e4239cd..812640374b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,6 +115,7 @@ IF (KokkosKernels_INSTALL_TESTING) KOKKOSKERNELS_ADD_TEST_DIRECTORIES(batched/dense/unit_test) KOKKOSKERNELS_ADD_TEST_DIRECTORIES(batched/sparse/unit_test) KOKKOSKERNELS_ADD_TEST_DIRECTORIES(blas/unit_test) + KOKKOSKERNELS_ADD_TEST_DIRECTORIES(lapack/unit_test) KOKKOSKERNELS_ADD_TEST_DIRECTORIES(graph/unit_test) KOKKOSKERNELS_ADD_TEST_DIRECTORIES(sparse/unit_test) KOKKOSKERNELS_ADD_TEST_DIRECTORIES(ode/unit_test) @@ -192,7 +193,7 @@ ELSE() "ALL" STRING "A list of components to enable in testing and building" - VALID_ENTRIES BATCHED BLAS GRAPH SPARSE ALL + VALID_ENTRIES BATCHED BLAS LAPACK GRAPH SPARSE ALL ) # ================================================================== @@ -243,6 +244,7 @@ ELSE() MESSAGE(" COMMON: ON") MESSAGE(" BATCHED: ${KokkosKernels_ENABLE_COMPONENT_BATCHED}") MESSAGE(" BLAS: ${KokkosKernels_ENABLE_COMPONENT_BLAS}") + MESSAGE(" LAPACK: ${KokkosKernels_ENABLE_COMPONENT_LAPACK}") MESSAGE(" GRAPH: ${KokkosKernels_ENABLE_COMPONENT_GRAPH}") MESSAGE(" SPARSE: ${KokkosKernels_ENABLE_COMPONENT_SPARSE}") MESSAGE(" ODE: ${KokkosKernels_ENABLE_COMPONENT_ODE}") @@ -287,6 +289,9 @@ ELSE() IF (KokkosKernels_ENABLE_COMPONENT_BLAS) INCLUDE(blas/CMakeLists.txt) ENDIF() + IF (KokkosKernels_ENABLE_COMPONENT_LAPACK) + INCLUDE(lapack/CMakeLists.txt) + ENDIF() IF (KokkosKernels_ENABLE_COMPONENT_GRAPH) INCLUDE(graph/CMakeLists.txt) ENDIF() @@ -405,6 +410,9 @@ ELSE() IF (KokkosKernels_ENABLE_COMPONENT_BLAS) KOKKOSKERNELS_ADD_TEST_DIRECTORIES(blas/unit_test) ENDIF() + IF (KokkosKernels_ENABLE_COMPONENT_LAPACK) + KOKKOSKERNELS_ADD_TEST_DIRECTORIES(lapack/unit_test) + ENDIF() IF (KokkosKernels_ENABLE_COMPONENT_GRAPH) KOKKOSKERNELS_ADD_TEST_DIRECTORIES(graph/unit_test) ENDIF() diff --git a/blas/CMakeLists.txt b/blas/CMakeLists.txt index 816d68e443..5bc7217cfd 100644 --- a/blas/CMakeLists.txt +++ b/blas/CMakeLists.txt @@ -101,13 +101,6 @@ KOKKOSKERNELS_GENERATE_ETI(Blas1_dot_mv dot TYPE_LISTS FLOATS LAYOUTS DEVICES ) -KOKKOSKERNELS_GENERATE_ETI(Blas_gesv gesv - COMPONENTS blas - HEADER_LIST ETI_HEADERS - SOURCE_LIST SOURCES - TYPE_LISTS FLOATS LAYOUTS DEVICES -) - KOKKOSKERNELS_GENERATE_ETI(Blas1_axpby axpby COMPONENTS blas HEADER_LIST ETI_HEADERS @@ -331,10 +324,3 @@ KOKKOSKERNELS_GENERATE_ETI(Blas3_trmm trmm SOURCE_LIST SOURCES TYPE_LISTS FLOATS LAYOUTS DEVICES ) - -KOKKOSKERNELS_GENERATE_ETI(Blas_trtri trtri - COMPONENTS blas - HEADER_LIST ETI_HEADERS - SOURCE_LIST SOURCES - TYPE_LISTS FLOATS LAYOUTS DEVICES -) diff --git a/blas/src/KokkosBlas_gesv.hpp b/blas/src/KokkosBlas_gesv.hpp index 89b9d36c96..1326c6fb8e 100644 --- a/blas/src/KokkosBlas_gesv.hpp +++ b/blas/src/KokkosBlas_gesv.hpp @@ -25,10 +25,7 @@ #ifndef KOKKOSBLAS_GESV_HPP_ #define KOKKOSBLAS_GESV_HPP_ -#include - -#include "KokkosBlas_gesv_spec.hpp" -#include "KokkosKernels_Error.hpp" +#include "KokkosLapack_gesv.hpp" namespace KokkosBlas { @@ -49,100 +46,8 @@ namespace KokkosBlas { /// its data pointer is NULL, pivoting is not used. /// template -void gesv(const AMatrix& A, const BXMV& B, const IPIVV& IPIV) { - // NOTE: Currently, KokkosBlas::gesv only supports for MAGMA TPL and BLAS TPL. - // MAGMA TPL should be enabled to call the MAGMA GPU interface for - // device views BLAS TPL should be enabled to call the BLAS interface - // for host views - - static_assert(Kokkos::is_view::value, - "KokkosBlas::gesv: A must be a Kokkos::View."); - static_assert(Kokkos::is_view::value, - "KokkosBlas::gesv: B must be a Kokkos::View."); - static_assert(Kokkos::is_view::value, - "KokkosBlas::gesv: IPIV must be a Kokkos::View."); - static_assert(static_cast(AMatrix::rank) == 2, - "KokkosBlas::gesv: A must have rank 2."); - static_assert( - static_cast(BXMV::rank) == 1 || static_cast(BXMV::rank) == 2, - "KokkosBlas::gesv: B must have either rank 1 or rank 2."); - static_assert(static_cast(IPIVV::rank) == 1, - "KokkosBlas::gesv: IPIV must have rank 1."); - - int64_t IPIV0 = IPIV.extent(0); - int64_t A0 = A.extent(0); - int64_t A1 = A.extent(1); - int64_t B0 = B.extent(0); - - // Check validity of pivot argument - bool valid_pivot = - (IPIV0 == A1) || ((IPIV0 == 0) && (IPIV.data() == nullptr)); - if (!(valid_pivot)) { - std::ostringstream os; - os << "KokkosBlas::gesv: IPIV: " << IPIV0 << ". " - << "Valid options include zero-extent 1-D view (no pivoting), or 1-D " - "View with size of " - << A0 << " (partial pivoting)."; - KokkosKernels::Impl::throw_runtime_exception(os.str()); - } - - // Check for no pivoting case. Only MAGMA supports no pivoting interface -#ifdef KOKKOSKERNELS_ENABLE_TPL_MAGMA // have MAGMA TPL -#ifdef KOKKOSKERNELS_ENABLE_TPL_BLAS // and have BLAS TPL - if ((!std::is_same::value) && - (IPIV0 == 0) && (IPIV.data() == nullptr)) { - std::ostringstream os; - os << "KokkosBlas::gesv: IPIV: " << IPIV0 << ". " - << "BLAS TPL does not support no pivoting."; - KokkosKernels::Impl::throw_runtime_exception(os.str()); - } -#endif -#else // not have MAGMA TPL -#ifdef KOKKOSKERNELS_ENABLE_TPL_BLAS // but have BLAS TPL - if ((IPIV0 == 0) && (IPIV.data() == nullptr)) { - std::ostringstream os; - os << "KokkosBlas::gesv: IPIV: " << IPIV0 << ". " - << "BLAS TPL does not support no pivoting."; - KokkosKernels::Impl::throw_runtime_exception(os.str()); - } -#endif -#endif - - // Check compatibility of dimensions at run time. - if ((A0 < A1) || (A0 != B0)) { - std::ostringstream os; - os << "KokkosBlas::gesv: Dimensions of A, and B do not match: " - << " A: " << A.extent(0) << " x " << A.extent(1) << " B: " << B.extent(0) - << " x " << B.extent(1); - KokkosKernels::Impl::throw_runtime_exception(os.str()); - } - - typedef Kokkos::View< - typename AMatrix::non_const_value_type**, typename AMatrix::array_layout, - typename AMatrix::device_type, Kokkos::MemoryTraits > - AMatrix_Internal; - typedef Kokkos::View > - BXMV_Internal; - typedef Kokkos::View< - typename IPIVV::non_const_value_type*, typename IPIVV::array_layout, - typename IPIVV::device_type, Kokkos::MemoryTraits > - IPIVV_Internal; - AMatrix_Internal A_i = A; - // BXMV_Internal B_i = B; - IPIVV_Internal IPIV_i = IPIV; - - if (BXMV::rank == 1) { - auto B_i = BXMV_Internal(B.data(), B.extent(0), 1); - KokkosBlas::Impl::GESV::gesv(A_i, B_i, IPIV_i); - } else { // BXMV::rank == 2 - auto B_i = BXMV_Internal(B.data(), B.extent(0), B.extent(1)); - KokkosBlas::Impl::GESV::gesv(A_i, B_i, IPIV_i); - } +[[deprecated]] void gesv(const AMatrix& A, const BXMV& B, const IPIVV& IPIV) { + KokkosLapack::gesv(A, B, IPIV); } } // namespace KokkosBlas diff --git a/blas/src/KokkosBlas_trtri.hpp b/blas/src/KokkosBlas_trtri.hpp index b1a34f0483..d9771e3a16 100644 --- a/blas/src/KokkosBlas_trtri.hpp +++ b/blas/src/KokkosBlas_trtri.hpp @@ -18,12 +18,7 @@ /// \file KokkosBlas_trtri.hpp -#include "KokkosKernels_Macros.hpp" -#include "KokkosBlas_trtri_spec.hpp" -#include "KokkosKernels_helpers.hpp" -#include -#include -#include "KokkosKernels_Error.hpp" +#include "KokkosLapack_trtri.hpp" namespace KokkosBlas { @@ -48,70 +43,9 @@ namespace KokkosBlas { // and the inversion could not be completed. // source: https://software.intel.com/en-us/mkl-developer-reference-c-trtri template -int trtri(const char uplo[], const char diag[], const AViewType& A) { - static_assert(Kokkos::is_view::value, - "AViewType must be a Kokkos::View."); - static_assert(static_cast(AViewType::rank) == 2, - "AViewType must have rank 2."); - - // Check validity of indicator argument - bool valid_uplo = (uplo[0] == 'U') || (uplo[0] == 'u') || (uplo[0] == 'L') || - (uplo[0] == 'l'); - bool valid_diag = (diag[0] == 'U') || (diag[0] == 'u') || (diag[0] == 'N') || - (diag[0] == 'n'); - - if (!valid_uplo) { - std::ostringstream os; - os << "KokkosBlas::trtri: uplo = '" << uplo[0] << "'. " - << "Valid values include 'U' or 'u' (A is upper triangular), " - "'L' or 'l' (A is lower triangular)."; - KokkosKernels::Impl::throw_runtime_exception(os.str()); - } - if (!valid_diag) { - std::ostringstream os; - os << "KokkosBlas::trtri: diag = '" << diag[0] << "'. " - << "Valid values include 'U' or 'u' (the diagonal of A is assumed to be " - "unit), " - "'N' or 'n' (the diagonal of A is assumed to be non-unit)."; - KokkosKernels::Impl::throw_runtime_exception(os.str()); - } - - int64_t A_m = A.extent(0); - int64_t A_n = A.extent(1); - - // Return if degenerated matrices are provided - if (A_m == 0 || A_n == 0) - return 0; // This is success as the inverse of a matrix with no elements is - // itself. - - // Ensure that the dimensions of A match and that we can legally perform A*B - // or B*A - if (A_m != A_n) { - std::ostringstream os; - os << "KokkosBlas::trtri: Dimensions of A do not match," - << " A: " << A.extent(0) << " x " << A.extent(1); - KokkosKernels::Impl::throw_runtime_exception(os.str()); - } - - // Create A matrix view type alias - using AViewInternalType = - Kokkos::View >; - - // This is the return value type and should always reside on host - using RViewInternalType = - Kokkos::View >; - - int result; - RViewInternalType R = RViewInternalType(&result); - - KokkosBlas::Impl::TRTRI::trtri(R, uplo, - diag, A); - - return result; +[[deprecated]] int trtri(const char uplo[], const char diag[], + const AViewType& A) { + return KokkosLapack::trtri(uplo, diag, A); } } // namespace KokkosBlas diff --git a/blas/tpls/KokkosBlas_Host_tpl.cpp b/blas/tpls/KokkosBlas_Host_tpl.cpp index a7be0d31ab..71e22a690c 100644 --- a/blas/tpls/KokkosBlas_Host_tpl.cpp +++ b/blas/tpls/KokkosBlas_Host_tpl.cpp @@ -411,39 +411,6 @@ void F77_BLAS_MANGLE(ztrsm, ZTRSM)(const char*, const char*, const char*, const std::complex*, const std::complex*, int*, /* */ std::complex*, int*); - -/// -/// Gesv -/// - -void F77_BLAS_MANGLE(sgesv, SGESV)(int*, int*, float*, int*, int*, float*, int*, - int*); -void F77_BLAS_MANGLE(dgesv, DGESV)(int*, int*, double*, int*, int*, double*, - int*, int*); -void F77_BLAS_MANGLE(cgesv, CGESV)(int*, int*, std::complex*, int*, int*, - std::complex*, int*, int*); -void F77_BLAS_MANGLE(zgesv, ZGESV)(int*, int*, std::complex*, int*, - int*, std::complex*, int*, int*); - -/// -/// Trtri -/// -/* - HostBlas::trtri(const char uplo, const char diag, - int n, const float *a, int lda) { - int info = 0; - F77_FUNC_STRTRI(&uplo, - &diag, &n, - a, &lda, &info); -*/ -void F77_BLAS_MANGLE(strtri, STRTRI)(const char*, const char*, int*, - const float*, int*, int*); -void F77_BLAS_MANGLE(dtrtri, DTRTRI)(const char*, const char*, int*, - const double*, int*, int*); -void F77_BLAS_MANGLE(ctrtri, CTRTRI)(const char*, const char*, int*, - const std::complex*, int*, int*); -void F77_BLAS_MANGLE(ztrtri, ZTRTRI)(const char*, const char*, int*, - const std::complex*, int*, int*); } void F77_BLAS_MANGLE(sscal, SSCAL)(const int* N, const float* alpha, @@ -559,16 +526,6 @@ void F77_BLAS_MANGLE(zscal, #define F77_FUNC_CTRSM F77_BLAS_MANGLE(ctrsm, CTRSM) #define F77_FUNC_ZTRSM F77_BLAS_MANGLE(ztrsm, ZTRSM) -#define F77_FUNC_SGESV F77_BLAS_MANGLE(sgesv, SGESV) -#define F77_FUNC_DGESV F77_BLAS_MANGLE(dgesv, DGESV) -#define F77_FUNC_CGESV F77_BLAS_MANGLE(cgesv, CGESV) -#define F77_FUNC_ZGESV F77_BLAS_MANGLE(zgesv, ZGESV) - -#define F77_FUNC_STRTRI F77_BLAS_MANGLE(strtri, STRTRI) -#define F77_FUNC_DTRTRI F77_BLAS_MANGLE(dtrtri, DTRTRI) -#define F77_FUNC_CTRTRI F77_BLAS_MANGLE(ctrtri, CTRTRI) -#define F77_FUNC_ZTRTRI F77_BLAS_MANGLE(ztrtri, ZTRTRI) - namespace KokkosBlas { namespace Impl { @@ -688,18 +645,6 @@ void HostBlas::trsm(const char side, const char uplo, const char transa, F77_FUNC_STRSM(&side, &uplo, &transa, &diag, &m, &n, &alpha, a, &lda, b, &ldb); } -template <> -void HostBlas::gesv(int n, int rhs, float* a, int lda, int* ipiv, - float* b, int ldb, int info) { - F77_FUNC_SGESV(&n, &rhs, a, &lda, ipiv, b, &ldb, &info); -} -template <> -int HostBlas::trtri(const char uplo, const char diag, int n, - const float* a, int lda) { - int info = 0; - F77_FUNC_STRTRI(&uplo, &diag, &n, a, &lda, &info); - return info; -} /// /// double @@ -818,18 +763,6 @@ void HostBlas::trsm(const char side, const char uplo, const char transa, F77_FUNC_DTRSM(&side, &uplo, &transa, &diag, &m, &n, &alpha, a, &lda, b, &ldb); } -template <> -void HostBlas::gesv(int n, int rhs, double* a, int lda, int* ipiv, - double* b, int ldb, int info) { - F77_FUNC_DGESV(&n, &rhs, a, &lda, ipiv, b, &ldb, &info); -} -template <> -int HostBlas::trtri(const char uplo, const char diag, int n, - const double* a, int lda) { - int info = 0; - F77_FUNC_DTRTRI(&uplo, &diag, &n, a, &lda, &info); - return info; -} /// /// std::complex @@ -1000,21 +933,6 @@ void HostBlas >::trsm(const char side, const char uplo, (const std::complex*)a, &lda, (std::complex*)b, &ldb); } -template <> -void HostBlas >::gesv(int n, int rhs, - std::complex* a, int lda, - int* ipiv, std::complex* b, - int ldb, int info) { - F77_FUNC_CGESV(&n, &rhs, a, &lda, ipiv, b, &ldb, &info); -} -template <> -int HostBlas >::trtri(const char uplo, const char diag, - int n, const std::complex* a, - int lda) { - int info = 0; - F77_FUNC_CTRTRI(&uplo, &diag, &n, a, &lda, &info); - return info; -} /// /// std::complex @@ -1183,21 +1101,6 @@ void HostBlas >::trsm( (const std::complex*)a, &lda, (std::complex*)b, &ldb); } -template <> -void HostBlas >::gesv(int n, int rhs, - std::complex* a, int lda, - int* ipiv, std::complex* b, - int ldb, int info) { - F77_FUNC_ZGESV(&n, &rhs, a, &lda, ipiv, b, &ldb, &info); -} -template <> -int HostBlas >::trtri(const char uplo, const char diag, - int n, const std::complex* a, - int lda) { - int info = 0; - F77_FUNC_ZTRTRI(&uplo, &diag, &n, a, &lda, &info); - return info; -} } // namespace Impl } // namespace KokkosBlas diff --git a/blas/tpls/KokkosBlas_Host_tpl.hpp b/blas/tpls/KokkosBlas_Host_tpl.hpp index 3b0c7f366e..29afff4d62 100644 --- a/blas/tpls/KokkosBlas_Host_tpl.hpp +++ b/blas/tpls/KokkosBlas_Host_tpl.hpp @@ -115,12 +115,6 @@ struct HostBlas { const char diag, int m, int n, const T alpha, const T *a, int lda, /* */ T *b, int ldb); - - static void gesv(int n, int rhs, T *a, int lda, int *ipiv, T *b, int ldb, - int info); - - static int trtri(const char uplo, const char diag, int n, const T *a, - int lda); }; } // namespace Impl } // namespace KokkosBlas diff --git a/blas/tpls/KokkosBlas_trtri_tpl_spec_avail.hpp b/blas/tpls/KokkosBlas_trtri_tpl_spec_avail.hpp deleted file mode 100644 index de9fc08c99..0000000000 --- a/blas/tpls/KokkosBlas_trtri_tpl_spec_avail.hpp +++ /dev/null @@ -1,107 +0,0 @@ -//@HEADER -// ************************************************************************ -// -// Kokkos v. 4.0 -// Copyright (2022) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. -// See https://kokkos.org/LICENSE for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//@HEADER - -#ifndef KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_HPP_ -#define KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_HPP_ - -namespace KokkosBlas { -namespace Impl { - -// Specialization struct which defines whether a specialization exists -template -struct trtri_tpl_spec_avail { - enum : bool { value = false }; -}; - -// Generic Host side LAPACK (could be MKL or whatever) -#define KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL(SCALAR, LAYOUTA, MEMSPACE) \ - template \ - struct trtri_tpl_spec_avail< \ - Kokkos::View >, \ - Kokkos::View, \ - Kokkos::MemoryTraits > > { \ - enum : bool { value = true }; \ - }; - -#ifdef KOKKOSKERNELS_ENABLE_TPL_BLAS -#define KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_BLAS(SCALAR, LAYOUTA, MEMSPACE) \ - KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL(SCALAR, LAYOUTA, MEMSPACE) -#else -#define KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_BLAS(SCALAR, LAYOUTA, MEMSPACE) -#endif // KOKKOSKERNELS_ENABLE_TPL_BLAS - -#ifdef KOKKOSKERNELS_ENABLE_TPL_MAGMA -#define KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(SCALAR, LAYOUTA, MEMSPACE) \ - KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL(SCALAR, LAYOUTA, MEMSPACE) -#else -#define KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(SCALAR, LAYOUTA, MEMSPACE) -#endif // KOKKOSKERNELS_ENABLE_TPL_MAGMA - -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_BLAS(double, Kokkos::LayoutLeft, - Kokkos::HostSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(double, Kokkos::LayoutLeft, - Kokkos::CudaSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(double, Kokkos::LayoutLeft, - Kokkos::CudaUVMSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_BLAS(float, Kokkos::LayoutLeft, - Kokkos::HostSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(float, Kokkos::LayoutLeft, - Kokkos::CudaSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(float, Kokkos::LayoutLeft, - Kokkos::CudaUVMSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_BLAS(Kokkos::complex, - Kokkos::LayoutLeft, Kokkos::HostSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, - Kokkos::LayoutLeft, Kokkos::CudaSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, - Kokkos::LayoutLeft, Kokkos::CudaUVMSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_BLAS(Kokkos::complex, Kokkos::LayoutLeft, - Kokkos::HostSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, - Kokkos::LayoutLeft, Kokkos::CudaSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, - Kokkos::LayoutLeft, Kokkos::CudaUVMSpace) - -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_BLAS(double, Kokkos::LayoutRight, - Kokkos::HostSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(double, Kokkos::LayoutRight, - Kokkos::CudaSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(double, Kokkos::LayoutRight, - Kokkos::CudaUVMSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_BLAS(float, Kokkos::LayoutRight, - Kokkos::HostSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(float, Kokkos::LayoutRight, - Kokkos::CudaSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(float, Kokkos::LayoutRight, - Kokkos::CudaUVMSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_BLAS(Kokkos::complex, - Kokkos::LayoutRight, Kokkos::HostSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, - Kokkos::LayoutRight, Kokkos::CudaSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, - Kokkos::LayoutRight, Kokkos::CudaUVMSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_BLAS(Kokkos::complex, - Kokkos::LayoutRight, Kokkos::HostSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, - Kokkos::LayoutRight, Kokkos::CudaSpace) -KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, - Kokkos::LayoutRight, Kokkos::CudaUVMSpace) - -} // namespace Impl -} // namespace KokkosBlas - -#endif // KOKKOSBLAS_TRTRI_TPL_SPEC_AVAIL_HPP_ diff --git a/blas/unit_test/Test_Blas.hpp b/blas/unit_test/Test_Blas.hpp index b370436391..9bb37d8d95 100644 --- a/blas/unit_test/Test_Blas.hpp +++ b/blas/unit_test/Test_Blas.hpp @@ -16,9 +16,6 @@ #ifndef TEST_BLAS_HPP #define TEST_BLAS_HPP -#include "Test_Blas_gesv.hpp" -#include "Test_Blas_trtri.hpp" - // Blas 1 #include "Test_Blas1_abs.hpp" #include "Test_Blas1_asum.hpp" diff --git a/cmake/KokkosKernels_config.h.in b/cmake/KokkosKernels_config.h.in index b8b66fffbb..7a61771231 100644 --- a/cmake/KokkosKernels_config.h.in +++ b/cmake/KokkosKernels_config.h.in @@ -109,6 +109,8 @@ /* BLAS library */ #cmakedefine KOKKOSKERNELS_ENABLE_TPL_BLAS +/* LAPACK */ +#cmakedefine KOKKOSKERNELS_ENABLE_TPL_LAPACK /* MKL library */ #cmakedefine KOKKOSKERNELS_ENABLE_TPL_MKL /* CUSPARSE */ diff --git a/cmake/kokkoskernels_components.cmake b/cmake/kokkoskernels_components.cmake index 1feb5bb8b8..16a784bd1f 100644 --- a/cmake/kokkoskernels_components.cmake +++ b/cmake/kokkoskernels_components.cmake @@ -29,6 +29,13 @@ KOKKOSKERNELS_ADD_OPTION( "Whether to build the blas component. Default: OFF" ) +KOKKOSKERNELS_ADD_OPTION( + "ENABLE_COMPONENT_LAPACK" + OFF + BOOL + "Whether to build the lapack component. Default: OFF" +) + # SPARSE depends on everything else at the moment. KOKKOSKERNELS_ADD_OPTION( "ENABLE_COMPONENT_SPARSE" @@ -67,6 +74,7 @@ ENDIF() IF (KokkosKernels_ENABLE_COMPONENT_SPARSE) SET(KokkosKernels_ENABLE_COMPONENT_BATCHED ON CACHE BOOL "" FORCE) SET(KokkosKernels_ENABLE_COMPONENT_BLAS ON CACHE BOOL "" FORCE) + SET(KokkosKernels_ENABLE_COMPONENT_LAPACK ON CACHE BOOL "" FORCE) SET(KokkosKernels_ENABLE_COMPONENT_GRAPH ON CACHE BOOL "" FORCE) ENDIF() @@ -74,6 +82,7 @@ ENDIF() IF (KokkosKernels_ENABLE_ALL_COMPONENTS) SET(KokkosKernels_ENABLE_COMPONENT_BATCHED ON CACHE BOOL "" FORCE) SET(KokkosKernels_ENABLE_COMPONENT_BLAS ON CACHE BOOL "" FORCE) + SET(KokkosKernels_ENABLE_COMPONENT_LAPACK ON CACHE BOOL "" FORCE) SET(KokkosKernels_ENABLE_COMPONENT_SPARSE ON CACHE BOOL "" FORCE) SET(KokkosKernels_ENABLE_COMPONENT_GRAPH ON CACHE BOOL "" FORCE) SET(KokkosKernels_ENABLE_COMPONENT_ODE ON CACHE BOOL "" FORCE) @@ -85,6 +94,7 @@ ENDIF() # but marking it as advanced should hide it from GUIs IF ( KokkosKernels_ENABLE_COMPONENT_BATCHED AND KokkosKernels_ENABLE_COMPONENT_BLAS + AND KokkosKernels_ENABLE_COMPONENT_LAPACK AND KokkosKernels_ENABLE_COMPONENT_GRAPH AND KokkosKernels_ENABLE_COMPONENT_SPARSE AND KokkosKernels_ENABLE_COMPONENT_ODE) diff --git a/lapack/CMakeLists.txt b/lapack/CMakeLists.txt new file mode 100644 index 0000000000..8ab784a325 --- /dev/null +++ b/lapack/CMakeLists.txt @@ -0,0 +1,67 @@ +LIST(APPEND KK_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/lapack/src) +LIST(APPEND KK_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/lapack/impl) +LIST(APPEND KK_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/lapack/eti) +LIST(APPEND KK_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/lapack/eti) +LIST(APPEND KK_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/lapack/tpls) + +# Adding unit-tests +KOKKOSKERNELS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/lapack) +KOKKOSKERNELS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}/lapack) + +######################### +# # +# Logic for LAPACK TPLs # +# # +######################### + +#Include LAPACK, Lapack host wrapper +IF (KOKKOSKERNELS_ENABLE_TPL_LAPACK OR KOKKOSKERNELS_ENABLE_TPL_MKL OR KOKKOSKERNELS_ENABLE_TPL_ARMPL) + #Do NOT add this to include path + APPEND_GLOB(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/tpls/KokkosLapack_Host_tpl.cpp) +ENDIF() + +# Include host lapack TPL source file +IF (KOKKOSKERNELS_ENABLE_TPL_LAPACK OR KOKKOSKERNELS_ENABLE_TPL_MKL OR KOKKOSKERNELS_ENABLE_TPL_ARMPL) + LIST(APPEND SOURCES + lapack/tpls/KokkosLapack_Host_tpl.cpp + ) +ENDIF() + +# Include cuda lapack TPL source file +IF (KOKKOSKERNELS_ENABLE_TPL_CUSOLVER) + LIST(APPEND SOURCES + lapack/tpls/KokkosLapack_Cuda_tpl.cpp + ) +ENDIF() + +# Include rocm lapack TPL source file +IF (KOKKOSKERNELS_ENABLE_TPL_ROCSOLVER) + LIST(APPEND SOURCES + lapack/tpls/KokkosLapack_Rocm_tpl.cpp + ) +ENDIF() + +################## +# # +# ETI generation # +# # +################## + +#Build up a list of DECL, AVAIL, and INST macros +#that should be instantiated based on input options +#Generate @X@ variables in the template X.hpp.in and X.cpp.in +#files containing the list of all needed macros + +KOKKOSKERNELS_GENERATE_ETI(Lapack_gesv gesv + COMPONENTS lapack + HEADER_LIST ETI_HEADERS + SOURCE_LIST SOURCES + TYPE_LISTS FLOATS LAYOUTS DEVICES +) + +KOKKOSKERNELS_GENERATE_ETI(Lapack_trtri trtri + COMPONENTS lapack + HEADER_LIST ETI_HEADERS + SOURCE_LIST SOURCES + TYPE_LISTS FLOATS LAYOUTS DEVICES +) diff --git a/blas/eti/generated_specializations_cpp/gesv/KokkosBlas_gesv_eti_spec_inst.cpp.in b/lapack/eti/generated_specializations_cpp/gesv/KokkosLapack_gesv_eti_spec_inst.cpp.in similarity index 88% rename from blas/eti/generated_specializations_cpp/gesv/KokkosBlas_gesv_eti_spec_inst.cpp.in rename to lapack/eti/generated_specializations_cpp/gesv/KokkosLapack_gesv_eti_spec_inst.cpp.in index 32473be3ad..da521984a4 100644 --- a/blas/eti/generated_specializations_cpp/gesv/KokkosBlas_gesv_eti_spec_inst.cpp.in +++ b/lapack/eti/generated_specializations_cpp/gesv/KokkosLapack_gesv_eti_spec_inst.cpp.in @@ -17,10 +17,10 @@ #define KOKKOSKERNELS_IMPL_COMPILE_LIBRARY true #include "KokkosKernels_config.h" -#include "KokkosBlas_gesv_spec.hpp" +#include "KokkosLapack_gesv_spec.hpp" -namespace KokkosBlas { +namespace KokkosLapack { namespace Impl { -@BLAS_GESV_ETI_INST_BLOCK@ +@LAPACK_GESV_ETI_INST_BLOCK@ } //IMPL } //Kokkos diff --git a/blas/eti/generated_specializations_cpp/trtri/KokkosBlas_trtri_eti_spec_inst.cpp.in b/lapack/eti/generated_specializations_cpp/trtri/KokkosLapack_trtri_eti_spec_inst.cpp.in similarity index 88% rename from blas/eti/generated_specializations_cpp/trtri/KokkosBlas_trtri_eti_spec_inst.cpp.in rename to lapack/eti/generated_specializations_cpp/trtri/KokkosLapack_trtri_eti_spec_inst.cpp.in index 64755f7a54..c4ab12f5a4 100644 --- a/blas/eti/generated_specializations_cpp/trtri/KokkosBlas_trtri_eti_spec_inst.cpp.in +++ b/lapack/eti/generated_specializations_cpp/trtri/KokkosLapack_trtri_eti_spec_inst.cpp.in @@ -17,10 +17,10 @@ #define KOKKOSKERNELS_IMPL_COMPILE_LIBRARY true #include "KokkosKernels_config.h" -#include "KokkosBlas_trtri_spec.hpp" +#include "KokkosLapack_trtri_spec.hpp" -namespace KokkosBlas { +namespace KokkosLapack { namespace Impl { -@BLAS_TRTRI_ETI_INST_BLOCK@ +@LAPACK_TRTRI_ETI_INST_BLOCK@ } //IMPL } //Kokkos diff --git a/blas/eti/generated_specializations_hpp/KokkosBlas_gesv_eti_spec_avail.hpp.in b/lapack/eti/generated_specializations_hpp/KokkosLapack_gesv_eti_spec_avail.hpp.in similarity index 80% rename from blas/eti/generated_specializations_hpp/KokkosBlas_gesv_eti_spec_avail.hpp.in rename to lapack/eti/generated_specializations_hpp/KokkosLapack_gesv_eti_spec_avail.hpp.in index ae262c912e..d1f36e3069 100644 --- a/blas/eti/generated_specializations_hpp/KokkosBlas_gesv_eti_spec_avail.hpp.in +++ b/lapack/eti/generated_specializations_hpp/KokkosLapack_gesv_eti_spec_avail.hpp.in @@ -14,11 +14,11 @@ // //@HEADER -#ifndef KOKKOSBLAS_GESV_ETI_SPEC_AVAIL_HPP_ -#define KOKKOSBLAS_GESV_ETI_SPEC_AVAIL_HPP_ -namespace KokkosBlas { +#ifndef KOKKOSLAPACK_GESV_ETI_SPEC_AVAIL_HPP_ +#define KOKKOSLAPACK_GESV_ETI_SPEC_AVAIL_HPP_ +namespace KokkosLapack { namespace Impl { -@BLAS_GESV_ETI_AVAIL_BLOCK@ +@LAPACK_GESV_ETI_AVAIL_BLOCK@ } //IMPL } //Kokkos #endif diff --git a/blas/eti/generated_specializations_hpp/KokkosBlas_trtri_eti_spec_avail.hpp.in b/lapack/eti/generated_specializations_hpp/KokkosLapack_trtri_eti_spec_avail.hpp.in similarity index 73% rename from blas/eti/generated_specializations_hpp/KokkosBlas_trtri_eti_spec_avail.hpp.in rename to lapack/eti/generated_specializations_hpp/KokkosLapack_trtri_eti_spec_avail.hpp.in index 3f669efa06..89443c2c9b 100644 --- a/blas/eti/generated_specializations_hpp/KokkosBlas_trtri_eti_spec_avail.hpp.in +++ b/lapack/eti/generated_specializations_hpp/KokkosLapack_trtri_eti_spec_avail.hpp.in @@ -14,13 +14,13 @@ // //@HEADER -#ifndef KOKKOSBLAS_TRTRI_ETI_SPEC_AVAIL_HPP_ -#define KOKKOSBLAS_TRTRI_ETI_SPEC_AVAIL_HPP_ -namespace KokkosBlas { +#ifndef KOKKOSLAPACK_TRTRI_ETI_SPEC_AVAIL_HPP_ +#define KOKKOSLAPACK_TRTRI_ETI_SPEC_AVAIL_HPP_ +namespace KokkosLapack { namespace Impl { -@BLAS_TRTRI_ETI_AVAIL_BLOCK@ +@LAPACK_TRTRI_ETI_AVAIL_BLOCK@ } // Impl -} // KokkosBlas -#endif // KOKKOSBLAS_TRTRI_ETI_SPEC_AVAIL_HPP_ +} // KokkosLapack +#endif // KOKKOSLAPACK_TRTRI_ETI_SPEC_AVAIL_HPP_ diff --git a/blas/impl/KokkosBlas_gesv_impl.hpp b/lapack/impl/KokkosLapack_gesv_impl.hpp similarity index 73% rename from blas/impl/KokkosBlas_gesv_impl.hpp rename to lapack/impl/KokkosLapack_gesv_impl.hpp index e51e48309f..3a60f42171 100644 --- a/blas/impl/KokkosBlas_gesv_impl.hpp +++ b/lapack/impl/KokkosLapack_gesv_impl.hpp @@ -14,21 +14,21 @@ // //@HEADER -#ifndef KOKKOSBLAS_IMPL_GESV_HPP_ -#define KOKKOSBLAS_IMPL_GESV_HPP_ +#ifndef KOKKOSLAPACK_IMPL_GESV_HPP_ +#define KOKKOSLAPACK_IMPL_GESV_HPP_ -/// \file KokkosBlas_gesv_impl.hpp +/// \file KokkosLapack_gesv_impl.hpp /// \brief Implementation(s) of dense linear solve. #include #include -namespace KokkosBlas { +namespace KokkosLapack { namespace Impl { -// NOTE: Might add the implementation of KokkosBlas::gesv later +// NOTE: Might add the implementation of KokkosLapack::gesv later } // namespace Impl -} // namespace KokkosBlas +} // namespace KokkosLapack -#endif // KOKKOSBLAS_IMPL_GESV_HPP +#endif // KOKKOSLAPACK_IMPL_GESV_HPP diff --git a/blas/impl/KokkosBlas_gesv_spec.hpp b/lapack/impl/KokkosLapack_gesv_spec.hpp similarity index 74% rename from blas/impl/KokkosBlas_gesv_spec.hpp rename to lapack/impl/KokkosLapack_gesv_spec.hpp index f1dff467c8..b9f8549311 100644 --- a/blas/impl/KokkosBlas_gesv_spec.hpp +++ b/lapack/impl/KokkosLapack_gesv_spec.hpp @@ -13,8 +13,8 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //@HEADER -#ifndef KOKKOSBLAS_IMPL_GESV_SPEC_HPP_ -#define KOKKOSBLAS_IMPL_GESV_SPEC_HPP_ +#ifndef KOKKOSLAPACK_IMPL_GESV_SPEC_HPP_ +#define KOKKOSLAPACK_IMPL_GESV_SPEC_HPP_ #include #include @@ -22,10 +22,10 @@ // Include the actual functors #if !defined(KOKKOSKERNELS_ETI_ONLY) || KOKKOSKERNELS_IMPL_COMPILE_LIBRARY -#include +#include #endif -namespace KokkosBlas { +namespace KokkosLapack { namespace Impl { // Specialization struct which defines whether a specialization exists template @@ -33,37 +33,37 @@ struct gesv_eti_spec_avail { enum : bool { value = false }; }; } // namespace Impl -} // namespace KokkosBlas +} // namespace KokkosLapack // // Macro for declaration of full specialization availability -// KokkosBlas::Impl::GESV. This is NOT for users!!! All +// KokkosLapack::Impl::GESV. This is NOT for users!!! All // the declarations of full specializations go in this header file. // We may spread out definitions (see _INST macro below) across one or // more .cpp files. // -#define KOKKOSBLAS_GESV_ETI_SPEC_AVAIL(SCALAR_TYPE, LAYOUT_TYPE, \ - EXEC_SPACE_TYPE, MEM_SPACE_TYPE) \ - template <> \ - struct gesv_eti_spec_avail< \ - Kokkos::View, \ - Kokkos::MemoryTraits >, \ - Kokkos::View, \ - Kokkos::MemoryTraits > > { \ - enum : bool { value = true }; \ +#define KOKKOSLAPACK_GESV_ETI_SPEC_AVAIL(SCALAR_TYPE, LAYOUT_TYPE, \ + EXEC_SPACE_TYPE, MEM_SPACE_TYPE) \ + template <> \ + struct gesv_eti_spec_avail< \ + Kokkos::View, \ + Kokkos::MemoryTraits >, \ + Kokkos::View, \ + Kokkos::MemoryTraits > > { \ + enum : bool { value = true }; \ }; // Include the actual specialization declarations -#include -#include +#include +#include -namespace KokkosBlas { +namespace KokkosLapack { namespace Impl { // Unification layer -/// \brief Implementation of KokkosBlas::gesv. +/// \brief Implementation of KokkosLapack::gesv. template ::value, @@ -79,54 +79,54 @@ template struct GESV { static void gesv(const AMatrix & /* A */, const BXMV & /* B */, const IPIVV & /* IPIV */) { - // NOTE: Might add the implementation of KokkosBlas::gesv later + // NOTE: Might add the implementation of KokkosLapack::gesv later throw std::runtime_error( "No fallback implementation of GESV (general LU factorization & solve) " - "exists. Enable BLAS and/or MAGMA TPL."); + "exists. Enable LAPACK and/or MAGMA TPL."); } }; #endif } // namespace Impl -} // namespace KokkosBlas +} // namespace KokkosLapack // // Macro for declaration of full specialization of -// KokkosBlas::Impl::GESV. This is NOT for users!!! All +// KokkosLapack::Impl::GESV. This is NOT for users!!! All // the declarations of full specializations go in this header file. // We may spread out definitions (see _DEF macro below) across one or // more .cpp files. // -#define KOKKOSBLAS_GESV_ETI_SPEC_DECL(SCALAR_TYPE, LAYOUT_TYPE, \ - EXEC_SPACE_TYPE, MEM_SPACE_TYPE) \ - extern template struct GESV< \ - Kokkos::View, \ - Kokkos::MemoryTraits >, \ - Kokkos::View, \ - Kokkos::MemoryTraits >, \ - Kokkos::View, \ - Kokkos::MemoryTraits >, \ +#define KOKKOSLAPACK_GESV_ETI_SPEC_DECL(SCALAR_TYPE, LAYOUT_TYPE, \ + EXEC_SPACE_TYPE, MEM_SPACE_TYPE) \ + extern template struct GESV< \ + Kokkos::View, \ + Kokkos::MemoryTraits >, \ + Kokkos::View, \ + Kokkos::MemoryTraits >, \ + Kokkos::View, \ + Kokkos::MemoryTraits >, \ false, true>; -#define KOKKOSBLAS_GESV_ETI_SPEC_INST(SCALAR_TYPE, LAYOUT_TYPE, \ - EXEC_SPACE_TYPE, MEM_SPACE_TYPE) \ - template struct GESV< \ - Kokkos::View, \ - Kokkos::MemoryTraits >, \ - Kokkos::View, \ - Kokkos::MemoryTraits >, \ - Kokkos::View, \ - Kokkos::MemoryTraits >, \ +#define KOKKOSLAPACK_GESV_ETI_SPEC_INST(SCALAR_TYPE, LAYOUT_TYPE, \ + EXEC_SPACE_TYPE, MEM_SPACE_TYPE) \ + template struct GESV< \ + Kokkos::View, \ + Kokkos::MemoryTraits >, \ + Kokkos::View, \ + Kokkos::MemoryTraits >, \ + Kokkos::View, \ + Kokkos::MemoryTraits >, \ false, true>; -#include +#include -#endif // KOKKOSBLAS_IMPL_GESV_SPEC_HPP_ +#endif // KOKKOSLAPACK_IMPL_GESV_SPEC_HPP_ diff --git a/blas/impl/KokkosBlas_trtri_impl.hpp b/lapack/impl/KokkosLapack_trtri_impl.hpp similarity index 91% rename from blas/impl/KokkosBlas_trtri_impl.hpp rename to lapack/impl/KokkosLapack_trtri_impl.hpp index 4501763ea8..9f52c2d412 100644 --- a/blas/impl/KokkosBlas_trtri_impl.hpp +++ b/lapack/impl/KokkosLapack_trtri_impl.hpp @@ -14,11 +14,11 @@ // //@HEADER -#ifndef KOKKOSBLAS_TRTRI_IMPL_HPP_ -#define KOKKOSBLAS_TRTRI_IMPL_HPP_ +#ifndef KOKKOSLAPACK_TRTRI_IMPL_HPP_ +#define KOKKOSLAPACK_TRTRI_IMPL_HPP_ /** - * \file KokkosBlas_trtri_impl.hpp + * \file KokkosLapack_trtri_impl.hpp * \brief Implementation of triangular matrix inverse */ @@ -27,7 +27,7 @@ #include "KokkosBatched_Trtri_Decl.hpp" #include "KokkosBatched_Trtri_Serial_Impl.hpp" -namespace KokkosBlas { +namespace KokkosLapack { namespace Impl { template @@ -65,5 +65,5 @@ void SerialTrtri_Invoke(const RViewType &R, const char uplo[], } } } // namespace Impl -} // namespace KokkosBlas -#endif // KOKKOSBLAS_TRTRI_IMPL_HPP_ +} // namespace KokkosLapack +#endif // KOKKOSLAPACK_TRTRI_IMPL_HPP_ diff --git a/blas/impl/KokkosBlas_trtri_spec.hpp b/lapack/impl/KokkosLapack_trtri_spec.hpp similarity index 77% rename from blas/impl/KokkosBlas_trtri_spec.hpp rename to lapack/impl/KokkosLapack_trtri_spec.hpp index 2a4d2db576..a17184dc41 100644 --- a/blas/impl/KokkosBlas_trtri_spec.hpp +++ b/lapack/impl/KokkosLapack_trtri_spec.hpp @@ -13,17 +13,17 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //@HEADER -#ifndef KOKKOSBLAS_TRTRI_SPEC_HPP_ -#define KOKKOSBLAS_TRTRI_SPEC_HPP_ +#ifndef KOKKOSLAPACK_TRTRI_SPEC_HPP_ +#define KOKKOSLAPACK_TRTRI_SPEC_HPP_ #include "KokkosKernels_config.h" #include "Kokkos_Core.hpp" #if !defined(KOKKOSKERNELS_ETI_ONLY) || KOKKOSKERNELS_IMPL_COMPILE_LIBRARY -#include +#include #endif -namespace KokkosBlas { +namespace KokkosLapack { namespace Impl { // Specialization struct which defines whether a specialization exists template @@ -31,14 +31,14 @@ struct trtri_eti_spec_avail { enum : bool { value = false }; }; } // namespace Impl -} // namespace KokkosBlas +} // namespace KokkosLapack // // This Macros provides the ETI specialization of trtri, currently not // available. // -#define KOKKOSBLAS_TRTRI_ETI_SPEC_AVAIL(SCALAR, LAYOUTA, EXEC_SPACE, \ - MEM_SPACE) \ +#define KOKKOSLAPACK_TRTRI_ETI_SPEC_AVAIL(SCALAR, LAYOUTA, EXEC_SPACE, \ + MEM_SPACE) \ template <> \ struct trtri_eti_spec_avail< \ Kokkos::View -#include +#include +#include -namespace KokkosBlas { +namespace KokkosLapack { namespace Impl { // @@ -77,8 +77,8 @@ struct TRTRI { static_assert(static_cast(AVIT::rank) == 2, "AVIT must have rank 2."); Kokkos::Profiling::pushRegion(KOKKOSKERNELS_IMPL_COMPILE_LIBRARY - ? "KokkosBlas::trtri[ETI]" - : "KokkosBlas::trtri[noETI]"); + ? "KokkosLapack::trtri[ETI]" + : "KokkosLapack::trtri[noETI]"); typename AVIT::HostMirror host_A = Kokkos::create_mirror_view(A); typename RVIT::HostMirror host_R = Kokkos::create_mirror_view(R); @@ -97,7 +97,7 @@ struct TRTRI { //! KOKKOSKERNELS_IMPL_COMPILE_LIBRARY } // namespace Impl -} // namespace KokkosBlas +} // namespace KokkosLapack // // These Macros are only included when we are not compiling libkokkoskernels but @@ -106,22 +106,24 @@ struct TRTRI { // "extern template" skips the implicit instatiation step ensuring that the // callers code uses this explicit instantiation definition of TRTRI. // -#define KOKKOSBLAS_TRTRI_ETI_SPEC_DECL(SCALAR, LAYOUTA, EXEC_SPACE, MEM_SPACE) \ - extern template struct TRTRI< \ - Kokkos::View >, \ - Kokkos::View, \ - Kokkos::MemoryTraits >, \ +#define KOKKOSLAPACK_TRTRI_ETI_SPEC_DECL(SCALAR, LAYOUTA, EXEC_SPACE, \ + MEM_SPACE) \ + extern template struct TRTRI< \ + Kokkos::View >, \ + Kokkos::View, \ + Kokkos::MemoryTraits >, \ false, true>; -#define KOKKOSBLAS_TRTRI_ETI_SPEC_INST(SCALAR, LAYOUTA, EXEC_SPACE, MEM_SPACE) \ - template struct TRTRI< \ - Kokkos::View >, \ - Kokkos::View, \ - Kokkos::MemoryTraits >, \ +#define KOKKOSLAPACK_TRTRI_ETI_SPEC_INST(SCALAR, LAYOUTA, EXEC_SPACE, \ + MEM_SPACE) \ + template struct TRTRI< \ + Kokkos::View >, \ + Kokkos::View, \ + Kokkos::MemoryTraits >, \ false, true>; -#include +#include -#endif // KOKKOSBLAS_TRTRI_SPEC_HPP_ +#endif // KOKKOSLAPACK_TRTRI_SPEC_HPP_ diff --git a/lapack/src/KokkosLapack_gesv.hpp b/lapack/src/KokkosLapack_gesv.hpp new file mode 100644 index 0000000000..4c9058f8ab --- /dev/null +++ b/lapack/src/KokkosLapack_gesv.hpp @@ -0,0 +1,151 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +/// \file KokkosLapack_gesv.hpp +/// \brief Local dense linear solve +/// +/// This file provides KokkosLapack::gesv. This function performs a +/// local (no MPI) dense linear solve on a system of linear equations +/// A * X = B where A is a general N-by-N matrix and X and B are N-by-NRHS +/// matrices. + +#ifndef KOKKOSLAPACK_GESV_HPP_ +#define KOKKOSLAPACK_GESV_HPP_ + +#include + +#include "KokkosLapack_gesv_spec.hpp" +#include "KokkosKernels_Error.hpp" + +namespace KokkosLapack { + +/// \brief Solve the dense linear equation system A*X = B. +/// +/// \tparam AMatrix Input matrix/Output LU, as a 2-D Kokkos::View. +/// \tparam BXMV Input (right-hand side)/Output (solution) (multi)vector, as a +/// 1-D or 2-D Kokkos::View. \tparam IPIVV Output pivot indices, as a 1-D +/// Kokkos::View +/// +/// \param A [in,out] On entry, the N-by-N matrix to be solved. On exit, the +/// factors L and U from +/// the factorization A = P*L*U; the unit diagonal elements of L are not +/// stored. +/// \param B [in,out] On entry, the right hand side (multi)vector B. On exit, +/// the solution (multi)vector X. \param IPIV [out] On exit, the pivot indices +/// (for partial pivoting). If the View extents are zero and +/// its data pointer is NULL, pivoting is not used. +/// +template +void gesv(const AMatrix& A, const BXMV& B, const IPIVV& IPIV) { + // NOTE: Currently, KokkosLapack::gesv only supports for MAGMA TPL and LAPACK + // TPL. + // MAGMA TPL should be enabled to call the MAGMA GPU interface for + // device views LAPACK TPL should be enabled to call the LAPACK + // interface for host views + + static_assert(Kokkos::is_view::value, + "KokkosLapack::gesv: A must be a Kokkos::View."); + static_assert(Kokkos::is_view::value, + "KokkosLapack::gesv: B must be a Kokkos::View."); + static_assert(Kokkos::is_view::value, + "KokkosLapack::gesv: IPIV must be a Kokkos::View."); + static_assert(static_cast(AMatrix::rank) == 2, + "KokkosLapack::gesv: A must have rank 2."); + static_assert( + static_cast(BXMV::rank) == 1 || static_cast(BXMV::rank) == 2, + "KokkosLapack::gesv: B must have either rank 1 or rank 2."); + static_assert(static_cast(IPIVV::rank) == 1, + "KokkosLapack::gesv: IPIV must have rank 1."); + + int64_t IPIV0 = IPIV.extent(0); + int64_t A0 = A.extent(0); + int64_t A1 = A.extent(1); + int64_t B0 = B.extent(0); + + // Check validity of pivot argument + bool valid_pivot = + (IPIV0 == A1) || ((IPIV0 == 0) && (IPIV.data() == nullptr)); + if (!(valid_pivot)) { + std::ostringstream os; + os << "KokkosLapack::gesv: IPIV: " << IPIV0 << ". " + << "Valid options include zero-extent 1-D view (no pivoting), or 1-D " + "View with size of " + << A0 << " (partial pivoting)."; + KokkosKernels::Impl::throw_runtime_exception(os.str()); + } + + // Check for no pivoting case. Only MAGMA supports no pivoting interface +#ifdef KOKKOSKERNELS_ENABLE_TPL_MAGMA // have MAGMA TPL +#ifdef KOKKOSKERNELS_ENABLE_TPL_LAPACK // and have LAPACK TPL + if ((!std::is_same::value) && + (IPIV0 == 0) && (IPIV.data() == nullptr)) { + std::ostringstream os; + os << "KokkosLapack::gesv: IPIV: " << IPIV0 << ". " + << "LAPACK TPL does not support no pivoting."; + KokkosKernels::Impl::throw_runtime_exception(os.str()); + } +#endif +#else // not have MAGMA TPL +#ifdef KOKKOSKERNELS_ENABLE_TPL_LAPACK // but have LAPACK TPL + if ((IPIV0 == 0) && (IPIV.data() == nullptr)) { + std::ostringstream os; + os << "KokkosLapack::gesv: IPIV: " << IPIV0 << ". " + << "LAPACK TPL does not support no pivoting."; + KokkosKernels::Impl::throw_runtime_exception(os.str()); + } +#endif +#endif + + // Check compatibility of dimensions at run time. + if ((A0 < A1) || (A0 != B0)) { + std::ostringstream os; + os << "KokkosLapack::gesv: Dimensions of A, and B do not match: " + << " A: " << A.extent(0) << " x " << A.extent(1) << " B: " << B.extent(0) + << " x " << B.extent(1); + KokkosKernels::Impl::throw_runtime_exception(os.str()); + } + + typedef Kokkos::View< + typename AMatrix::non_const_value_type**, typename AMatrix::array_layout, + typename AMatrix::device_type, Kokkos::MemoryTraits > + AMatrix_Internal; + typedef Kokkos::View > + BXMV_Internal; + typedef Kokkos::View< + typename IPIVV::non_const_value_type*, typename IPIVV::array_layout, + typename IPIVV::device_type, Kokkos::MemoryTraits > + IPIVV_Internal; + AMatrix_Internal A_i = A; + // BXMV_Internal B_i = B; + IPIVV_Internal IPIV_i = IPIV; + + if (BXMV::rank == 1) { + auto B_i = BXMV_Internal(B.data(), B.extent(0), 1); + KokkosLapack::Impl::GESV::gesv(A_i, B_i, IPIV_i); + } else { // BXMV::rank == 2 + auto B_i = BXMV_Internal(B.data(), B.extent(0), B.extent(1)); + KokkosLapack::Impl::GESV::gesv(A_i, B_i, IPIV_i); + } +} + +} // namespace KokkosLapack + +#endif // KOKKOSLAPACK_GESV_HPP_ diff --git a/lapack/src/KokkosLapack_trtri.hpp b/lapack/src/KokkosLapack_trtri.hpp new file mode 100644 index 0000000000..9a884f2303 --- /dev/null +++ b/lapack/src/KokkosLapack_trtri.hpp @@ -0,0 +1,119 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER +#ifndef KOKKOSLAPACK_TRTRI_HPP_ +#define KOKKOSLAPACK_TRTRI_HPP_ + +/// \file KokkosLapack_trtri.hpp + +#include "KokkosKernels_Macros.hpp" +#include "KokkosLapack_trtri_spec.hpp" +#include "KokkosKernels_helpers.hpp" +#include +#include +#include "KokkosKernels_Error.hpp" + +namespace KokkosLapack { + +/// \brief Find the inverse of the triangular matrix, A +/// +/// A = inv(A) +/// +/// \tparam AViewType Input matrix, as a 2-D Kokkos::View +/// +/// \param uplo [in] "U" or "u" indicates matrix A is an upper triangular +/// matrix +/// "L" or "l" indicates matrix A is a lower triangular matrix +/// \param diag [in] "U" or "u" indicates the diagonal of A is assumed to be +/// unit +// "N" or "n" indicates the diagonal of A is assumed to be +// non-unit +/// \param A [in,out] Input matrix, as a 2-D Kokkos::View +/// On entry, A +/// On successful exit, inv(A) +/// \return 0 upon success, +// i if the i-th diagonal elemet of A is zero, A is singular, +// and the inversion could not be completed. +// source: https://software.intel.com/en-us/mkl-developer-reference-c-trtri +template +int trtri(const char uplo[], const char diag[], const AViewType& A) { + static_assert(Kokkos::is_view::value, + "AViewType must be a Kokkos::View."); + static_assert(static_cast(AViewType::rank) == 2, + "AViewType must have rank 2."); + + // Check validity of indicator argument + bool valid_uplo = (uplo[0] == 'U') || (uplo[0] == 'u') || (uplo[0] == 'L') || + (uplo[0] == 'l'); + bool valid_diag = (diag[0] == 'U') || (diag[0] == 'u') || (diag[0] == 'N') || + (diag[0] == 'n'); + + if (!valid_uplo) { + std::ostringstream os; + os << "KokkosLapack::trtri: uplo = '" << uplo[0] << "'. " + << "Valid values include 'U' or 'u' (A is upper triangular), " + "'L' or 'l' (A is lower triangular)."; + KokkosKernels::Impl::throw_runtime_exception(os.str()); + } + if (!valid_diag) { + std::ostringstream os; + os << "KokkosLapack::trtri: diag = '" << diag[0] << "'. " + << "Valid values include 'U' or 'u' (the diagonal of A is assumed to be " + "unit), " + "'N' or 'n' (the diagonal of A is assumed to be non-unit)."; + KokkosKernels::Impl::throw_runtime_exception(os.str()); + } + + int64_t A_m = A.extent(0); + int64_t A_n = A.extent(1); + + // Return if degenerated matrices are provided + if (A_m == 0 || A_n == 0) + return 0; // This is success as the inverse of a matrix with no elements is + // itself. + + // Ensure that the dimensions of A match and that we can legally perform A*B + // or B*A + if (A_m != A_n) { + std::ostringstream os; + os << "KokkosLapack::trtri: Dimensions of A do not match," + << " A: " << A.extent(0) << " x " << A.extent(1); + KokkosKernels::Impl::throw_runtime_exception(os.str()); + } + + // Create A matrix view type alias + using AViewInternalType = + Kokkos::View >; + + // This is the return value type and should always reside on host + using RViewInternalType = + Kokkos::View >; + + int result; + RViewInternalType R = RViewInternalType(&result); + + KokkosLapack::Impl::TRTRI::trtri( + R, uplo, diag, A); + + return result; +} + +} // namespace KokkosLapack + +#endif // KOKKOSLAPACK_TRTRI_HPP_ diff --git a/lapack/tpls/KokkosLapack_Cuda_tpl.cpp b/lapack/tpls/KokkosLapack_Cuda_tpl.cpp new file mode 100644 index 0000000000..2ac28871a4 --- /dev/null +++ b/lapack/tpls/KokkosLapack_Cuda_tpl.cpp @@ -0,0 +1,18 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER +#include +#include +#include diff --git a/lapack/tpls/KokkosLapack_Cuda_tpl.hpp b/lapack/tpls/KokkosLapack_Cuda_tpl.hpp new file mode 100644 index 0000000000..b59d6d99c8 --- /dev/null +++ b/lapack/tpls/KokkosLapack_Cuda_tpl.hpp @@ -0,0 +1,64 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER +#ifndef KOKKOSLAPACK_CUDA_TPL_HPP_ +#define KOKKOSLAPACK_CUDA_TPL_HPP_ + +#if defined(KOKKOSKERNELS_ENABLE_TPL_CUSOLVER) +#include + +namespace KokkosLapack { +namespace Impl { + +CudaLapackSingleton::CudaLapackSingleton() { + cusolverStatus_t stat = cusolverDnCreate(&handle); + if (stat != CUSOLVER_STATUS_SUCCESS) + Kokkos::abort("CUSOLVER initialization failed\n"); + + Kokkos::push_finalize_hook([&]() { cusolverDnDestroy(handle); }); +} + +CudaLapackSingleton& CudaLapackSingleton::singleton() { + static CudaLapackSingleton s; + return s; +} + +} // namespace Impl +} // namespace KokkosLapack +#endif // defined (KOKKOSKERNELS_ENABLE_TPL_CUSOLVER) + +#if defined(KOKKOSKERNELS_ENABLE_TPL_MAGMA) +#include + +namespace KokkosLapack { +namespace Impl { + +MagmaSingleton::MagmaSingleton() { + magma_int_t stat = magma_init(); + if (stat != MAGMA_SUCCESS) Kokkos::abort("MAGMA initialization failed\n"); + + Kokkos::push_finalize_hook([&]() { magma_finalize(); }); +} + +MagmaSingleton& MagmaSingleton::singleton() { + static MagmaSingleton s; + return s; +} + +} // namespace Impl +} // namespace KokkosLapack +#endif // defined(KOKKOSKERNELS_ENABLE_TPL_MAGMA) + +#endif // KOKKOSLAPACK_CUDA_TPL_HPP_ diff --git a/lapack/tpls/KokkosLapack_Host_tpl.cpp b/lapack/tpls/KokkosLapack_Host_tpl.cpp new file mode 100644 index 0000000000..d629a17f1d --- /dev/null +++ b/lapack/tpls/KokkosLapack_Host_tpl.cpp @@ -0,0 +1,152 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER +/// \file KokkosLapack_Host_tpl.cpp +/// \brief LAPACK wrapper for host tpls +/// \author Kyungjoo Kim (kyukim@sandia.gov) + +#include "KokkosKernels_config.h" +#include "KokkosLapack_Host_tpl.hpp" + +#if defined(KOKKOSKERNELS_ENABLE_TPL_LAPACK) + +/// Fortran headers +extern "C" { + +/// +/// Gesv +/// + +void F77_BLAS_MANGLE(sgesv, SGESV)(int*, int*, float*, int*, int*, float*, int*, + int*); +void F77_BLAS_MANGLE(dgesv, DGESV)(int*, int*, double*, int*, int*, double*, + int*, int*); +void F77_BLAS_MANGLE(cgesv, CGESV)(int*, int*, std::complex*, int*, int*, + std::complex*, int*, int*); +void F77_BLAS_MANGLE(zgesv, ZGESV)(int*, int*, std::complex*, int*, + int*, std::complex*, int*, int*); + +/// +/// Trtri +/// +/* + HostLapack::trtri(const char uplo, const char diag, + int n, const float *a, int lda) { + int info = 0; + F77_FUNC_STRTRI(&uplo, + &diag, &n, + a, &lda, &info); +*/ +void F77_BLAS_MANGLE(strtri, STRTRI)(const char*, const char*, int*, + const float*, int*, int*); +void F77_BLAS_MANGLE(dtrtri, DTRTRI)(const char*, const char*, int*, + const double*, int*, int*); +void F77_BLAS_MANGLE(ctrtri, CTRTRI)(const char*, const char*, int*, + const std::complex*, int*, int*); +void F77_BLAS_MANGLE(ztrtri, ZTRTRI)(const char*, const char*, int*, + const std::complex*, int*, int*); +} + +#define F77_FUNC_SGESV F77_BLAS_MANGLE(sgesv, SGESV) +#define F77_FUNC_DGESV F77_BLAS_MANGLE(dgesv, DGESV) +#define F77_FUNC_CGESV F77_BLAS_MANGLE(cgesv, CGESV) +#define F77_FUNC_ZGESV F77_BLAS_MANGLE(zgesv, ZGESV) + +#define F77_FUNC_STRTRI F77_BLAS_MANGLE(strtri, STRTRI) +#define F77_FUNC_DTRTRI F77_BLAS_MANGLE(dtrtri, DTRTRI) +#define F77_FUNC_CTRTRI F77_BLAS_MANGLE(ctrtri, CTRTRI) +#define F77_FUNC_ZTRTRI F77_BLAS_MANGLE(ztrtri, ZTRTRI) + +namespace KokkosLapack { +namespace Impl { + +/// +/// float +/// + +template <> +void HostLapack::gesv(int n, int rhs, float* a, int lda, int* ipiv, + float* b, int ldb, int info) { + F77_FUNC_SGESV(&n, &rhs, a, &lda, ipiv, b, &ldb, &info); +} +template <> +int HostLapack::trtri(const char uplo, const char diag, int n, + const float* a, int lda) { + int info = 0; + F77_FUNC_STRTRI(&uplo, &diag, &n, a, &lda, &info); + return info; +} + +/// +/// double +/// + +template <> +void HostLapack::gesv(int n, int rhs, double* a, int lda, int* ipiv, + double* b, int ldb, int info) { + F77_FUNC_DGESV(&n, &rhs, a, &lda, ipiv, b, &ldb, &info); +} +template <> +int HostLapack::trtri(const char uplo, const char diag, int n, + const double* a, int lda) { + int info = 0; + F77_FUNC_DTRTRI(&uplo, &diag, &n, a, &lda, &info); + return info; +} + +/// +/// std::complex +/// + +template <> +void HostLapack >::gesv(int n, int rhs, + std::complex* a, int lda, + int* ipiv, std::complex* b, + int ldb, int info) { + F77_FUNC_CGESV(&n, &rhs, a, &lda, ipiv, b, &ldb, &info); +} +template <> +int HostLapack >::trtri(const char uplo, const char diag, + int n, const std::complex* a, + int lda) { + int info = 0; + F77_FUNC_CTRTRI(&uplo, &diag, &n, a, &lda, &info); + return info; +} + +/// +/// std::complex +/// + +template <> +void HostLapack >::gesv(int n, int rhs, + std::complex* a, int lda, + int* ipiv, std::complex* b, + int ldb, int info) { + F77_FUNC_ZGESV(&n, &rhs, a, &lda, ipiv, b, &ldb, &info); +} +template <> +int HostLapack >::trtri(const char uplo, const char diag, + int n, + const std::complex* a, + int lda) { + int info = 0; + F77_FUNC_ZTRTRI(&uplo, &diag, &n, a, &lda, &info); + return info; +} + +} // namespace Impl +} // namespace KokkosLapack +#endif // KOKKOSKERNELS_ENABLE_TPL_LAPACK diff --git a/lapack/tpls/KokkosLapack_Host_tpl.hpp b/lapack/tpls/KokkosLapack_Host_tpl.hpp new file mode 100644 index 0000000000..d74099aaec --- /dev/null +++ b/lapack/tpls/KokkosLapack_Host_tpl.hpp @@ -0,0 +1,44 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOSLAPACK_HOST_TPL_HPP_ +#define KOKKOSLAPACK_HOST_TPL_HPP_ + +/// \file KokkosLapack_Host_tpl.hpp +/// \brief LAPACK wrapper + +#include "KokkosKernels_config.h" +#include "Kokkos_ArithTraits.hpp" + +#if defined(KOKKOSKERNELS_ENABLE_TPL_LAPACK) + +namespace KokkosLapack { +namespace Impl { + +template +struct HostLapack { + static void gesv(int n, int rhs, T *a, int lda, int *ipiv, T *b, int ldb, + int info); + + static int trtri(const char uplo, const char diag, int n, const T *a, + int lda); +}; +} // namespace Impl +} // namespace KokkosLapack + +#endif // KOKKOSKERNELS_ENABLE_TPL_LAPACK + +#endif // KOKKOSLAPACK_HOST_TPL_HPP_ diff --git a/blas/tpls/KokkosBlas_gesv_tpl_spec_avail.hpp b/lapack/tpls/KokkosLapack_gesv_tpl_spec_avail.hpp similarity index 60% rename from blas/tpls/KokkosBlas_gesv_tpl_spec_avail.hpp rename to lapack/tpls/KokkosLapack_gesv_tpl_spec_avail.hpp index f909b4a295..a3d8bb6ee9 100644 --- a/blas/tpls/KokkosBlas_gesv_tpl_spec_avail.hpp +++ b/lapack/tpls/KokkosLapack_gesv_tpl_spec_avail.hpp @@ -14,10 +14,10 @@ // //@HEADER -#ifndef KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_HPP_ -#define KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_HPP_ +#ifndef KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_HPP_ +#define KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_HPP_ -namespace KokkosBlas { +namespace KokkosLapack { namespace Impl { // Specialization struct which defines whether a specialization exists template @@ -25,10 +25,10 @@ struct gesv_tpl_spec_avail { enum : bool { value = false }; }; -// Generic Host side BLAS (could be MKL or whatever) -#ifdef KOKKOSKERNELS_ENABLE_TPL_BLAS +// Generic Host side LAPACK (could be MKL or whatever) +#ifdef KOKKOSKERNELS_ENABLE_TPL_LAPACK -#define KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_BLAS(SCALAR, LAYOUT, MEMSPACE) \ +#define KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_LAPACK(SCALAR, LAYOUT, MEMSPACE) \ template \ struct gesv_tpl_spec_avail< \ Kokkos::View, \ @@ -38,30 +38,30 @@ struct gesv_tpl_spec_avail { enum : bool { value = true }; \ }; -KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_BLAS(double, Kokkos::LayoutLeft, - Kokkos::HostSpace) -KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_BLAS(float, Kokkos::LayoutLeft, - Kokkos::HostSpace) -KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_BLAS(Kokkos::complex, Kokkos::LayoutLeft, - Kokkos::HostSpace) -KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_BLAS(Kokkos::complex, Kokkos::LayoutLeft, - Kokkos::HostSpace) +KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_LAPACK(double, Kokkos::LayoutLeft, + Kokkos::HostSpace) +KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_LAPACK(float, Kokkos::LayoutLeft, + Kokkos::HostSpace) +KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_LAPACK(Kokkos::complex, + Kokkos::LayoutLeft, Kokkos::HostSpace) +KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_LAPACK(Kokkos::complex, + Kokkos::LayoutLeft, Kokkos::HostSpace) /* #if defined (KOKKOSKERNELS_INST_DOUBLE) \ && defined (KOKKOSKERNELS_INST_LAYOUTRIGHT) - KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_BLAS( double, Kokkos::LayoutRight, + KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_LAPACK( double, Kokkos::LayoutRight, Kokkos::HostSpace) #endif #if defined (KOKKOSKERNELS_INST_FLOAT) \ && defined (KOKKOSKERNELS_INST_LAYOUTRIGHT) - KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_BLAS( float, Kokkos::LayoutRight, + KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_LAPACK( float, Kokkos::LayoutRight, Kokkos::HostSpace) #endif #if defined (KOKKOSKERNELS_INST_KOKKOS_COMPLEX_DOUBLE_) \ && defined (KOKKOSKERNELS_INST_LAYOUTRIGHT) - KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_BLAS( Kokkos::complex, + KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_LAPACK( Kokkos::complex, Kokkos::LayoutRight, Kokkos::HostSpace) #endif #if defined (KOKKOSKERNELS_INST_KOKKOS_COMPLEX_FLOAT_) \ && defined (KOKKOSKERNELS_INST_LAYOUTRIGHT) - KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_BLAS( Kokkos::complex, + KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_LAPACK( Kokkos::complex, Kokkos::LayoutRight, Kokkos::HostSpace) #endif */ #endif @@ -69,7 +69,7 @@ Kokkos::LayoutRight, Kokkos::HostSpace) #endif // MAGMA #ifdef KOKKOSKERNELS_ENABLE_TPL_MAGMA -#define KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_MAGMA(SCALAR, LAYOUT, MEMSPACE) \ +#define KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_MAGMA(SCALAR, LAYOUT, MEMSPACE) \ template \ struct gesv_tpl_spec_avail< \ Kokkos::View, \ @@ -79,36 +79,36 @@ Kokkos::LayoutRight, Kokkos::HostSpace) #endif enum : bool { value = true }; \ }; -KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_MAGMA(double, Kokkos::LayoutLeft, - Kokkos::CudaSpace) -KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_MAGMA(float, Kokkos::LayoutLeft, - Kokkos::CudaSpace) -KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, - Kokkos::LayoutLeft, Kokkos::CudaSpace) -KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, Kokkos::LayoutLeft, - Kokkos::CudaSpace) +KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_MAGMA(double, Kokkos::LayoutLeft, + Kokkos::CudaSpace) +KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_MAGMA(float, Kokkos::LayoutLeft, + Kokkos::CudaSpace) +KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, + Kokkos::LayoutLeft, Kokkos::CudaSpace) +KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, + Kokkos::LayoutLeft, Kokkos::CudaSpace) /* #if defined (KOKKOSKERNELS_INST_DOUBLE) \ && defined (KOKKOSKERNELS_INST_LAYOUTRIGHT) - KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_MAGMA( double, Kokkos::LayoutRight, + KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_MAGMA( double, Kokkos::LayoutRight, Kokkos::CudaSpace) #endif #if defined (KOKKOSKERNELS_INST_FLOAT) \ && defined (KOKKOSKERNELS_INST_LAYOUTRIGHT) - KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_MAGMA( float, Kokkos::LayoutRight, + KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_MAGMA( float, Kokkos::LayoutRight, Kokkos::CudaSpace) #endif #if defined (KOKKOSKERNELS_INST_KOKKOS_COMPLEX_DOUBLE_) \ && defined (KOKKOSKERNELS_INST_LAYOUTRIGHT) - KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_MAGMA( + KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_MAGMA( Kokkos::complex,Kokkos::LayoutRight, Kokkos::CudaSpace) #endif #if defined (KOKKOSKERNELS_INST_KOKKOS_COMPLEX_FLOAT_) \ && defined (KOKKOSKERNELS_INST_LAYOUTRIGHT) - KOKKOSBLAS_GESV_TPL_SPEC_AVAIL_MAGMA( Kokkos::complex, + KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_MAGMA( Kokkos::complex, Kokkos::LayoutRight, Kokkos::CudaSpace) #endif */ #endif } // namespace Impl -} // namespace KokkosBlas +} // namespace KokkosLapack #endif diff --git a/blas/tpls/KokkosBlas_gesv_tpl_spec_decl.hpp b/lapack/tpls/KokkosLapack_gesv_tpl_spec_decl.hpp similarity index 87% rename from blas/tpls/KokkosBlas_gesv_tpl_spec_decl.hpp rename to lapack/tpls/KokkosLapack_gesv_tpl_spec_decl.hpp index 7d8f0a8a2b..2baa76a132 100644 --- a/blas/tpls/KokkosBlas_gesv_tpl_spec_decl.hpp +++ b/lapack/tpls/KokkosLapack_gesv_tpl_spec_decl.hpp @@ -14,21 +14,21 @@ // //@HEADER -#ifndef KOKKOSBLAS_GESV_TPL_SPEC_DECL_HPP_ -#define KOKKOSBLAS_GESV_TPL_SPEC_DECL_HPP_ +#ifndef KOKKOSLAPACK_GESV_TPL_SPEC_DECL_HPP_ +#define KOKKOSLAPACK_GESV_TPL_SPEC_DECL_HPP_ -namespace KokkosBlas { +namespace KokkosLapack { namespace Impl { template inline void gesv_print_specialization() { #ifdef KOKKOSKERNELS_ENABLE_CHECK_SPECIALIZATION #ifdef KOKKOSKERNELS_ENABLE_TPL_MAGMA - printf("KokkosBlas::gesv<> TPL MAGMA specialization for < %s , %s, %s >\n", + printf("KokkosLapack::gesv<> TPL MAGMA specialization for < %s , %s, %s >\n", typeid(AViewType).name(), typeid(BViewType).name(), typeid(PViewType).name()); #else -#ifdef KOKKOSKERNELS_ENABLE_TPL_BLAS - printf("KokkosBlas::gesv<> TPL Blas specialization for < %s , %s, %s >\n", +#ifdef KOKKOSKERNELS_ENABLE_TPL_LAPACK + printf("KokkosLapack::gesv<> TPL Lapack specialization for < %s , %s, %s >\n", typeid(AViewType).name(), typeid(BViewType).name(), typeid(PViewType).name()); #endif @@ -36,16 +36,16 @@ inline void gesv_print_specialization() { #endif } } // namespace Impl -} // namespace KokkosBlas +} // namespace KokkosLapack -// Generic Host side BLAS (could be MKL or whatever) -#ifdef KOKKOSKERNELS_ENABLE_TPL_BLAS -#include +// Generic Host side LAPACK (could be MKL or whatever) +#ifdef KOKKOSKERNELS_ENABLE_TPL_LAPACK +#include -namespace KokkosBlas { +namespace KokkosLapack { namespace Impl { -#define KOKKOSBLAS_DGESV_BLAS(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ +#define KOKKOSLAPACK_DGESV_LAPACK(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ template \ struct GESV< \ Kokkos::View, \ @@ -74,7 +74,7 @@ namespace Impl { \ static void gesv(const AViewType& A, const BViewType& B, \ const PViewType& IPIV) { \ - Kokkos::Profiling::pushRegion("KokkosBlas::gesv[TPL_BLAS,double]"); \ + Kokkos::Profiling::pushRegion("KokkosLapack::gesv[TPL_LAPACK,double]"); \ gesv_print_specialization(); \ const bool with_pivot = \ !((IPIV.extent(0) == 0) && (IPIV.data() == nullptr)); \ @@ -89,65 +89,65 @@ namespace Impl { int info = 0; \ \ if (with_pivot) { \ - HostBlas::gesv(N, NRHS, A.data(), LDA, IPIV.data(), B.data(), \ - LDB, info); \ + HostLapack::gesv(N, NRHS, A.data(), LDA, IPIV.data(), \ + B.data(), LDB, info); \ } \ Kokkos::Profiling::popRegion(); \ } \ }; -#define KOKKOSBLAS_SGESV_BLAS(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ - template \ - struct GESV< \ - Kokkos::View, \ - Kokkos::MemoryTraits >, \ - Kokkos::View, \ - Kokkos::MemoryTraits >, \ - Kokkos::View, \ - Kokkos::MemoryTraits >, \ - true, ETI_SPEC_AVAIL> { \ - typedef float SCALAR; \ - typedef Kokkos::View, \ - Kokkos::MemoryTraits > \ - AViewType; \ - typedef Kokkos::View, \ - Kokkos::MemoryTraits > \ - BViewType; \ - typedef Kokkos::View< \ - int*, LAYOUT, \ - Kokkos::Device, \ - Kokkos::MemoryTraits > \ - PViewType; \ - \ - static void gesv(const AViewType& A, const BViewType& B, \ - const PViewType& IPIV) { \ - Kokkos::Profiling::pushRegion("KokkosBlas::gesv[TPL_BLAS,float]"); \ - gesv_print_specialization(); \ - const bool with_pivot = \ - !((IPIV.extent(0) == 0) && (IPIV.data() == nullptr)); \ - \ - const int N = static_cast(A.extent(1)); \ - const int AST = static_cast(A.stride(1)); \ - const int LDA = (AST == 0) ? 1 : AST; \ - const int BST = static_cast(B.stride(1)); \ - const int LDB = (BST == 0) ? 1 : BST; \ - const int NRHS = static_cast(B.extent(1)); \ - \ - int info = 0; \ - \ - if (with_pivot) { \ - HostBlas::gesv(N, NRHS, A.data(), LDA, IPIV.data(), B.data(), \ - LDB, info); \ - } \ - Kokkos::Profiling::popRegion(); \ - } \ +#define KOKKOSLAPACK_SGESV_LAPACK(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ + template \ + struct GESV< \ + Kokkos::View, \ + Kokkos::MemoryTraits >, \ + Kokkos::View, \ + Kokkos::MemoryTraits >, \ + Kokkos::View, \ + Kokkos::MemoryTraits >, \ + true, ETI_SPEC_AVAIL> { \ + typedef float SCALAR; \ + typedef Kokkos::View, \ + Kokkos::MemoryTraits > \ + AViewType; \ + typedef Kokkos::View, \ + Kokkos::MemoryTraits > \ + BViewType; \ + typedef Kokkos::View< \ + int*, LAYOUT, \ + Kokkos::Device, \ + Kokkos::MemoryTraits > \ + PViewType; \ + \ + static void gesv(const AViewType& A, const BViewType& B, \ + const PViewType& IPIV) { \ + Kokkos::Profiling::pushRegion("KokkosLapack::gesv[TPL_LAPACK,float]"); \ + gesv_print_specialization(); \ + const bool with_pivot = \ + !((IPIV.extent(0) == 0) && (IPIV.data() == nullptr)); \ + \ + const int N = static_cast(A.extent(1)); \ + const int AST = static_cast(A.stride(1)); \ + const int LDA = (AST == 0) ? 1 : AST; \ + const int BST = static_cast(B.stride(1)); \ + const int LDB = (BST == 0) ? 1 : BST; \ + const int NRHS = static_cast(B.extent(1)); \ + \ + int info = 0; \ + \ + if (with_pivot) { \ + HostLapack::gesv(N, NRHS, A.data(), LDA, IPIV.data(), B.data(), \ + LDB, info); \ + } \ + Kokkos::Profiling::popRegion(); \ + } \ }; -#define KOKKOSBLAS_ZGESV_BLAS(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ +#define KOKKOSLAPACK_ZGESV_LAPACK(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ template \ struct GESV**, LAYOUT, \ Kokkos::Device, \ @@ -178,7 +178,7 @@ namespace Impl { static void gesv(const AViewType& A, const BViewType& B, \ const PViewType& IPIV) { \ Kokkos::Profiling::pushRegion( \ - "KokkosBlas::gesv[TPL_BLAS,complex]"); \ + "KokkosLapack::gesv[TPL_LAPACK,complex]"); \ gesv_print_specialization(); \ const bool with_pivot = \ !((IPIV.extent(0) == 0) && (IPIV.data() == nullptr)); \ @@ -193,7 +193,7 @@ namespace Impl { int info = 0; \ \ if (with_pivot) { \ - HostBlas >::gesv( \ + HostLapack >::gesv( \ N, NRHS, reinterpret_cast*>(A.data()), LDA, \ IPIV.data(), reinterpret_cast*>(B.data()), \ LDB, info); \ @@ -202,7 +202,7 @@ namespace Impl { } \ }; -#define KOKKOSBLAS_CGESV_BLAS(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ +#define KOKKOSLAPACK_CGESV_LAPACK(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ template \ struct GESV**, LAYOUT, \ Kokkos::Device, \ @@ -233,7 +233,7 @@ namespace Impl { static void gesv(const AViewType& A, const BViewType& B, \ const PViewType& IPIV) { \ Kokkos::Profiling::pushRegion( \ - "KokkosBlas::gesv[TPL_BLAS,complex]"); \ + "KokkosLapack::gesv[TPL_LAPACK,complex]"); \ gesv_print_specialization(); \ const bool with_pivot = \ !((IPIV.extent(0) == 0) && (IPIV.data() == nullptr)); \ @@ -248,7 +248,7 @@ namespace Impl { int info = 0; \ \ if (with_pivot) { \ - HostBlas >::gesv( \ + HostLapack >::gesv( \ N, NRHS, reinterpret_cast*>(A.data()), LDA, \ IPIV.data(), reinterpret_cast*>(B.data()), \ LDB, info); \ @@ -257,30 +257,30 @@ namespace Impl { } \ }; -KOKKOSBLAS_DGESV_BLAS(Kokkos::LayoutLeft, Kokkos::HostSpace, true) -KOKKOSBLAS_DGESV_BLAS(Kokkos::LayoutLeft, Kokkos::HostSpace, false) +KOKKOSLAPACK_DGESV_LAPACK(Kokkos::LayoutLeft, Kokkos::HostSpace, true) +KOKKOSLAPACK_DGESV_LAPACK(Kokkos::LayoutLeft, Kokkos::HostSpace, false) -KOKKOSBLAS_SGESV_BLAS(Kokkos::LayoutLeft, Kokkos::HostSpace, true) -KOKKOSBLAS_SGESV_BLAS(Kokkos::LayoutLeft, Kokkos::HostSpace, false) +KOKKOSLAPACK_SGESV_LAPACK(Kokkos::LayoutLeft, Kokkos::HostSpace, true) +KOKKOSLAPACK_SGESV_LAPACK(Kokkos::LayoutLeft, Kokkos::HostSpace, false) -KOKKOSBLAS_ZGESV_BLAS(Kokkos::LayoutLeft, Kokkos::HostSpace, true) -KOKKOSBLAS_ZGESV_BLAS(Kokkos::LayoutLeft, Kokkos::HostSpace, false) +KOKKOSLAPACK_ZGESV_LAPACK(Kokkos::LayoutLeft, Kokkos::HostSpace, true) +KOKKOSLAPACK_ZGESV_LAPACK(Kokkos::LayoutLeft, Kokkos::HostSpace, false) -KOKKOSBLAS_CGESV_BLAS(Kokkos::LayoutLeft, Kokkos::HostSpace, true) -KOKKOSBLAS_CGESV_BLAS(Kokkos::LayoutLeft, Kokkos::HostSpace, false) +KOKKOSLAPACK_CGESV_LAPACK(Kokkos::LayoutLeft, Kokkos::HostSpace, true) +KOKKOSLAPACK_CGESV_LAPACK(Kokkos::LayoutLeft, Kokkos::HostSpace, false) } // namespace Impl -} // namespace KokkosBlas -#endif // KOKKOSKERNELS_ENABLE_TPL_BLAS +} // namespace KokkosLapack +#endif // KOKKOSKERNELS_ENABLE_TPL_LAPACK // MAGMA #ifdef KOKKOSKERNELS_ENABLE_TPL_MAGMA -#include +#include -namespace KokkosBlas { +namespace KokkosLapack { namespace Impl { -#define KOKKOSBLAS_DGESV_MAGMA(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ +#define KOKKOSLAPACK_DGESV_MAGMA(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ template \ struct GESV< \ Kokkos::View, \ @@ -309,7 +309,7 @@ namespace Impl { \ static void gesv(const AViewType& A, const BViewType& B, \ const PViewType& IPIV) { \ - Kokkos::Profiling::pushRegion("KokkosBlas::gesv[TPL_MAGMA,double]"); \ + Kokkos::Profiling::pushRegion("KokkosLapack::gesv[TPL_MAGMA,double]"); \ gesv_print_specialization(); \ const bool with_pivot = \ !((IPIV.extent(0) == 0) && (IPIV.data() == nullptr)); \ @@ -321,8 +321,8 @@ namespace Impl { magma_int_t LDB = (BST == 0) ? 1 : BST; \ magma_int_t NRHS = static_cast(B.extent(1)); \ \ - KokkosBlas::Impl::MagmaSingleton& s = \ - KokkosBlas::Impl::MagmaSingleton::singleton(); \ + KokkosLapack::Impl::MagmaSingleton& s = \ + KokkosLapack::Impl::MagmaSingleton::singleton(); \ magma_int_t info = 0; \ \ if (with_pivot) { \ @@ -339,7 +339,7 @@ namespace Impl { } \ }; -#define KOKKOSBLAS_SGESV_MAGMA(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ +#define KOKKOSLAPACK_SGESV_MAGMA(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ template \ struct GESV< \ Kokkos::View, \ @@ -368,7 +368,7 @@ namespace Impl { \ static void gesv(const AViewType& A, const BViewType& B, \ const PViewType& IPIV) { \ - Kokkos::Profiling::pushRegion("KokkosBlas::gesv[TPL_MAGMA,float]"); \ + Kokkos::Profiling::pushRegion("KokkosLapack::gesv[TPL_MAGMA,float]"); \ gesv_print_specialization(); \ const bool with_pivot = \ !((IPIV.extent(0) == 0) && (IPIV.data() == nullptr)); \ @@ -380,8 +380,8 @@ namespace Impl { magma_int_t LDB = (BST == 0) ? 1 : BST; \ magma_int_t NRHS = static_cast(B.extent(1)); \ \ - KokkosBlas::Impl::MagmaSingleton& s = \ - KokkosBlas::Impl::MagmaSingleton::singleton(); \ + KokkosLapack::Impl::MagmaSingleton& s = \ + KokkosLapack::Impl::MagmaSingleton::singleton(); \ magma_int_t info = 0; \ \ if (with_pivot) { \ @@ -398,7 +398,7 @@ namespace Impl { } \ }; -#define KOKKOSBLAS_ZGESV_MAGMA(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ +#define KOKKOSLAPACK_ZGESV_MAGMA(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ template \ struct GESV**, LAYOUT, \ Kokkos::Device, \ @@ -429,7 +429,7 @@ namespace Impl { static void gesv(const AViewType& A, const BViewType& B, \ const PViewType& IPIV) { \ Kokkos::Profiling::pushRegion( \ - "KokkosBlas::gesv[TPL_MAGMA,complex]"); \ + "KokkosLapack::gesv[TPL_MAGMA,complex]"); \ gesv_print_specialization(); \ const bool with_pivot = \ !((IPIV.extent(0) == 0) && (IPIV.data() == nullptr)); \ @@ -441,8 +441,8 @@ namespace Impl { magma_int_t LDB = (BST == 0) ? 1 : BST; \ magma_int_t NRHS = static_cast(B.extent(1)); \ \ - KokkosBlas::Impl::MagmaSingleton& s = \ - KokkosBlas::Impl::MagmaSingleton::singleton(); \ + KokkosLapack::Impl::MagmaSingleton& s = \ + KokkosLapack::Impl::MagmaSingleton::singleton(); \ magma_int_t info = 0; \ \ if (with_pivot) { \ @@ -459,7 +459,7 @@ namespace Impl { } \ }; -#define KOKKOSBLAS_CGESV_MAGMA(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ +#define KOKKOSLAPACK_CGESV_MAGMA(LAYOUT, MEM_SPACE, ETI_SPEC_AVAIL) \ template \ struct GESV**, LAYOUT, \ Kokkos::Device, \ @@ -490,7 +490,7 @@ namespace Impl { static void gesv(const AViewType& A, const BViewType& B, \ const PViewType& IPIV) { \ Kokkos::Profiling::pushRegion( \ - "KokkosBlas::gesv[TPL_MAGMA,complex]"); \ + "KokkosLapack::gesv[TPL_MAGMA,complex]"); \ gesv_print_specialization(); \ const bool with_pivot = \ !((IPIV.extent(0) == 0) && (IPIV.data() == nullptr)); \ @@ -502,8 +502,8 @@ namespace Impl { magma_int_t LDB = (BST == 0) ? 1 : BST; \ magma_int_t NRHS = static_cast(B.extent(1)); \ \ - KokkosBlas::Impl::MagmaSingleton& s = \ - KokkosBlas::Impl::MagmaSingleton::singleton(); \ + KokkosLapack::Impl::MagmaSingleton& s = \ + KokkosLapack::Impl::MagmaSingleton::singleton(); \ magma_int_t info = 0; \ \ if (with_pivot) { \ @@ -520,20 +520,20 @@ namespace Impl { } \ }; -KOKKOSBLAS_DGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, true) -KOKKOSBLAS_DGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, false) +KOKKOSLAPACK_DGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, true) +KOKKOSLAPACK_DGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, false) -KOKKOSBLAS_SGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, true) -KOKKOSBLAS_SGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, false) +KOKKOSLAPACK_SGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, true) +KOKKOSLAPACK_SGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, false) -KOKKOSBLAS_ZGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, true) -KOKKOSBLAS_ZGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, false) +KOKKOSLAPACK_ZGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, true) +KOKKOSLAPACK_ZGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, false) -KOKKOSBLAS_CGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, true) -KOKKOSBLAS_CGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, false) +KOKKOSLAPACK_CGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, true) +KOKKOSLAPACK_CGESV_MAGMA(Kokkos::LayoutLeft, Kokkos::CudaSpace, false) } // namespace Impl -} // namespace KokkosBlas +} // namespace KokkosLapack #endif // KOKKOSKERNELS_ENABLE_TPL_MAGMA #endif diff --git a/lapack/tpls/KokkosLapack_trtri_tpl_spec_avail.hpp b/lapack/tpls/KokkosLapack_trtri_tpl_spec_avail.hpp new file mode 100644 index 0000000000..7251d97086 --- /dev/null +++ b/lapack/tpls/KokkosLapack_trtri_tpl_spec_avail.hpp @@ -0,0 +1,133 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_HPP_ +#define KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_HPP_ + +namespace KokkosLapack { +namespace Impl { + +// Specialization struct which defines whether a specialization exists +template +struct trtri_tpl_spec_avail { + enum : bool { value = false }; +}; + +// Generic Host side LAPACK (could be MKL or whatever) +#define KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL(SCALAR, LAYOUTA, MEMSPACE) \ + template \ + struct trtri_tpl_spec_avail< \ + Kokkos::View >, \ + Kokkos::View, \ + Kokkos::MemoryTraits > > { \ + enum : bool { value = true }; \ + }; + +#ifdef KOKKOSKERNELS_ENABLE_TPL_LAPACK +#define KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_LAPACK(SCALAR, LAYOUTA, MEMSPACE) \ + KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL(SCALAR, LAYOUTA, MEMSPACE) +#else +#define KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_LAPACK(SCALAR, LAYOUTA, MEMSPACE) +#endif // KOKKOSKERNELS_ENABLE_TPL_LAPACK + +#ifdef KOKKOSKERNELS_ENABLE_TPL_MAGMA +#define KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(SCALAR, LAYOUTA, MEMSPACE) \ + KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL(SCALAR, LAYOUTA, MEMSPACE) +#else +#define KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(SCALAR, LAYOUTA, MEMSPACE) +#endif // KOKKOSKERNELS_ENABLE_TPL_MAGMA + +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_LAPACK(double, Kokkos::LayoutLeft, + Kokkos::HostSpace) +#ifdef KOKKOS_ENABLE_CUDA +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(double, Kokkos::LayoutLeft, + Kokkos::CudaSpace) +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(double, Kokkos::LayoutLeft, + Kokkos::CudaUVMSpace) +#endif + +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_LAPACK(float, Kokkos::LayoutLeft, + Kokkos::HostSpace) +#ifdef KOKKOS_ENABLE_CUDA +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(float, Kokkos::LayoutLeft, + Kokkos::CudaSpace) +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(float, Kokkos::LayoutLeft, + Kokkos::CudaUVMSpace) +#endif + +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_LAPACK(Kokkos::complex, + Kokkos::LayoutLeft, Kokkos::HostSpace) +#ifdef KOKKOS_ENABLE_CUDA +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, + Kokkos::LayoutLeft, Kokkos::CudaSpace) +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, + Kokkos::LayoutLeft, + Kokkos::CudaUVMSpace) +#endif + +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_LAPACK(Kokkos::complex, + Kokkos::LayoutLeft, Kokkos::HostSpace) +#ifdef KOKKOS_ENABLE_CUDA +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, + Kokkos::LayoutLeft, Kokkos::CudaSpace) +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, + Kokkos::LayoutLeft, + Kokkos::CudaUVMSpace) +#endif + +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_LAPACK(double, Kokkos::LayoutRight, + Kokkos::HostSpace) +#ifdef KOKKOS_ENABLE_CUDA +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(double, Kokkos::LayoutRight, + Kokkos::CudaSpace) +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(double, Kokkos::LayoutRight, + Kokkos::CudaUVMSpace) +#endif + +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_LAPACK(float, Kokkos::LayoutRight, + Kokkos::HostSpace) +#ifdef KOKKOS_ENABLE_CUDA +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(float, Kokkos::LayoutRight, + Kokkos::CudaSpace) +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(float, Kokkos::LayoutRight, + Kokkos::CudaUVMSpace) +#endif + +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_LAPACK(Kokkos::complex, + Kokkos::LayoutRight, Kokkos::HostSpace) +#ifdef KOKKOS_ENABLE_CUDA +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, + Kokkos::LayoutRight, Kokkos::CudaSpace) +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, + Kokkos::LayoutRight, + Kokkos::CudaUVMSpace) +#endif + +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_LAPACK(Kokkos::complex, + Kokkos::LayoutRight, Kokkos::HostSpace) +#ifdef KOKKOS_ENABLE_CUDA +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, + Kokkos::LayoutRight, Kokkos::CudaSpace) +KOKKOSLAPACK_TRTRI_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex, + Kokkos::LayoutRight, + Kokkos::CudaUVMSpace) +#endif + +} // namespace Impl +} // namespace KokkosLapack + +#endif // KOKKOSLAPACKy_TRTRI_TPL_SPEC_AVAIL_HPP_ diff --git a/blas/tpls/KokkosBlas_trtri_tpl_spec_decl.hpp b/lapack/tpls/KokkosLapack_trtri_tpl_spec_decl.hpp similarity index 50% rename from blas/tpls/KokkosBlas_trtri_tpl_spec_decl.hpp rename to lapack/tpls/KokkosLapack_trtri_tpl_spec_decl.hpp index 46ec894547..3ed0623018 100644 --- a/blas/tpls/KokkosBlas_trtri_tpl_spec_decl.hpp +++ b/lapack/tpls/KokkosLapack_trtri_tpl_spec_decl.hpp @@ -14,18 +14,18 @@ // //@HEADER -#ifndef KOKKOSBLAS_TRTRI_TPL_SPEC_DECL_HPP_ -#define KOKKOSBLAS_TRTRI_TPL_SPEC_DECL_HPP_ +#ifndef KOKKOSLAPACK_TRTRI_TPL_SPEC_DECL_HPP_ +#define KOKKOSLAPACK_TRTRI_TPL_SPEC_DECL_HPP_ -#include "KokkosBlas_Host_tpl.hpp" // trtri prototype -#include "KokkosBlas_tpl_spec.hpp" +#include "KokkosLapack_Host_tpl.hpp" // trtri prototype +//#include "KokkosLapack_tpl_spec.hpp" -namespace KokkosBlas { +namespace KokkosLapack { namespace Impl { -#ifdef KOKKOSKERNELS_ENABLE_TPL_BLAS -#define KOKKOSBLAS_TRTRI_BLAS_HOST(SCALAR_TYPE, BASE_SCALAR_TYPE, LAYOUTA, \ - MEM_SPACE, ETI_SPEC_AVAIL) \ +#ifdef KOKKOSKERNELS_ENABLE_TPL_LAPACK +#define KOKKOSLAPACK_TRTRI_LAPACK_HOST(SCALAR_TYPE, BASE_SCALAR_TYPE, LAYOUTA, \ + MEM_SPACE, ETI_SPEC_AVAIL) \ template \ struct TRTRI >, \ @@ -44,8 +44,8 @@ namespace Impl { \ static void trtri(const RViewType& R, const char uplo[], \ const char diag[], const AViewType& A) { \ - Kokkos::Profiling::pushRegion("KokkosBlas::trtri[TPL_BLAS," #SCALAR_TYPE \ - "]"); \ + Kokkos::Profiling::pushRegion( \ + "KokkosLapack::trtri[TPL_LAPACK," #SCALAR_TYPE "]"); \ const int M = static_cast(A.extent(0)); \ \ bool A_is_layout_left = \ @@ -61,136 +61,164 @@ namespace Impl { else \ uplo_ = A_is_layout_left ? 'U' : 'L'; \ \ - R() = HostBlas::trtri( \ + R() = HostLapack::trtri( \ uplo_, diag[0], M, \ reinterpret_cast(A.data()), LDA); \ Kokkos::Profiling::popRegion(); \ } \ }; #else -#define KOKKOSBLAS_TRTRI_BLAS_HOST(SCALAR_TYPE, BASE_SCALAR_TYPE, LAYOUTA, \ - MEM_SPACE, ETI_SPEC_AVAIL) -#endif // KOKKOSKERNELS_ENABLE_TPL_BLAS +#define KOKKOSLAPACK_TRTRI_LAPACK_HOST(SCALAR_TYPE, BASE_SCALAR_TYPE, LAYOUTA, \ + MEM_SPACE, ETI_SPEC_AVAIL) +#endif // KOKKOSKERNELS_ENABLE_TPL_LAPACK #ifdef KOKKOSKERNELS_ENABLE_TPL_MAGMA -#define KOKKOSBLAS_TRTRI_BLAS_MAGMA(SCALAR_TYPE, BASE_SCALAR_TYPE, MAGMA_FN, \ - LAYOUTA, MEM_SPACE, ETI_SPEC_AVAIL) \ - template \ - struct TRTRI >, \ - Kokkos::View, \ - Kokkos::MemoryTraits >, \ - true, ETI_SPEC_AVAIL> { \ - typedef SCALAR_TYPE SCALAR; \ - typedef Kokkos::View > \ - RViewType; \ - typedef Kokkos::View, \ - Kokkos::MemoryTraits > \ - AViewType; \ - \ - static void trtri(const RViewType& R, const char uplo[], \ - const char diag[], const AViewType& A) { \ - Kokkos::Profiling::pushRegion("KokkosBlas::trtri[TPL_BLAS," #SCALAR_TYPE \ - "]"); \ - magma_int_t M = static_cast(A.extent(0)); \ - \ - bool A_is_layout_left = \ - std::is_same::value; \ - \ - magma_int_t AST = A_is_layout_left ? A.stride(1) : A.stride(0), \ - LDA = (AST == 0) ? 1 : AST; \ - magma_int_t info = 0; \ - magma_uplo_t uplo_; \ - magma_diag_t diag_; \ - \ - if ((uplo[0] == 'L') || (uplo[0] == 'l')) \ - uplo_ = A_is_layout_left ? MagmaLower : MagmaUpper; \ - else \ - uplo_ = A_is_layout_left ? MagmaUpper : MagmaLower; \ - \ - if (diag[0] == 'U' || diag[0] == 'u') \ - diag_ = MagmaUnit; \ - else \ - diag_ = MagmaNonUnit; \ - \ - KokkosBlas::Impl::MagmaSingleton& s = \ - KokkosBlas::Impl::MagmaSingleton::singleton(); \ - R() = MAGMA_FN(uplo_, diag_, M, \ - reinterpret_cast( \ - const_cast(A.data())), \ - LDA, &info); \ - Kokkos::Profiling::popRegion(); \ - } \ +#define KOKKOSLAPACK_TRTRI_LAPACK_MAGMA(SCALAR_TYPE, BASE_SCALAR_TYPE, \ + MAGMA_FN, LAYOUTA, MEM_SPACE, \ + ETI_SPEC_AVAIL) \ + template \ + struct TRTRI >, \ + Kokkos::View, \ + Kokkos::MemoryTraits >, \ + true, ETI_SPEC_AVAIL> { \ + typedef SCALAR_TYPE SCALAR; \ + typedef Kokkos::View > \ + RViewType; \ + typedef Kokkos::View, \ + Kokkos::MemoryTraits > \ + AViewType; \ + \ + static void trtri(const RViewType& R, const char uplo[], \ + const char diag[], const AViewType& A) { \ + Kokkos::Profiling::pushRegion( \ + "KokkosLapack::trtri[TPL_LAPACK," #SCALAR_TYPE "]"); \ + magma_int_t M = static_cast(A.extent(0)); \ + \ + bool A_is_layout_left = \ + std::is_same::value; \ + \ + magma_int_t AST = A_is_layout_left ? A.stride(1) : A.stride(0), \ + LDA = (AST == 0) ? 1 : AST; \ + magma_int_t info = 0; \ + magma_uplo_t uplo_; \ + magma_diag_t diag_; \ + \ + if ((uplo[0] == 'L') || (uplo[0] == 'l')) \ + uplo_ = A_is_layout_left ? MagmaLower : MagmaUpper; \ + else \ + uplo_ = A_is_layout_left ? MagmaUpper : MagmaLower; \ + \ + if (diag[0] == 'U' || diag[0] == 'u') \ + diag_ = MagmaUnit; \ + else \ + diag_ = MagmaNonUnit; \ + \ + KokkosLapack::Impl::MagmaSingleton& s = \ + KokkosLapack::Impl::MagmaSingleton::singleton(); \ + R() = MAGMA_FN(uplo_, diag_, M, \ + reinterpret_cast( \ + const_cast(A.data())), \ + LDA, &info); \ + Kokkos::Profiling::popRegion(); \ + } \ }; #else -#define KOKKOSBLAS_TRTRI_BLAS_MAGMA(SCALAR_TYPE, BASE_SCALAR_TYPE, MAGMA_FN, \ - LAYOUTA, MEM_SPACE, ETI_SPEC_AVAIL) +#define KOKKOSLAPACK_TRTRI_LAPACK_MAGMA(SCALAR_TYPE, BASE_SCALAR_TYPE, \ + MAGMA_FN, LAYOUTA, MEM_SPACE, \ + ETI_SPEC_AVAIL) #endif // KOKKOSKERNELS_ENABLE_TPL_MAGMA // Explicitly define the TRTRI class for all permutations listed below // Handle type and space permutations -#define KOKKOSBLAS_DTRTRI_BLAS(LAYOUTA, ETI_SPEC_AVAIL) \ - KOKKOSBLAS_TRTRI_BLAS_HOST(double, double, LAYOUTA, Kokkos::HostSpace, \ - ETI_SPEC_AVAIL) \ - KOKKOSBLAS_TRTRI_BLAS_MAGMA(double, magmaDouble_ptr, magma_dtrtri_gpu, \ - LAYOUTA, Kokkos::CudaSpace, ETI_SPEC_AVAIL) \ - KOKKOSBLAS_TRTRI_BLAS_MAGMA(double, magmaDouble_ptr, magma_dtrtri_gpu, \ - LAYOUTA, Kokkos::CudaUVMSpace, ETI_SPEC_AVAIL) - -#define KOKKOSBLAS_STRTRI_BLAS(LAYOUTA, ETI_SPEC_AVAIL) \ - KOKKOSBLAS_TRTRI_BLAS_HOST(float, float, LAYOUTA, Kokkos::HostSpace, \ - ETI_SPEC_AVAIL) \ - KOKKOSBLAS_TRTRI_BLAS_MAGMA(float, magmaFloat_ptr, magma_strtri_gpu, \ - LAYOUTA, Kokkos::CudaSpace, ETI_SPEC_AVAIL) \ - KOKKOSBLAS_TRTRI_BLAS_MAGMA(float, magmaFloat_ptr, magma_strtri_gpu, \ - LAYOUTA, Kokkos::CudaUVMSpace, ETI_SPEC_AVAIL) - -#define KOKKOSBLAS_ZTRTRI_BLAS(LAYOUTA, ETI_SPEC_AVAIL) \ - KOKKOSBLAS_TRTRI_BLAS_HOST(Kokkos::complex, std::complex, \ - LAYOUTA, Kokkos::HostSpace, ETI_SPEC_AVAIL) \ - KOKKOSBLAS_TRTRI_BLAS_MAGMA(Kokkos::complex, magmaDoubleComplex_ptr, \ - magma_ztrtri_gpu, LAYOUTA, Kokkos::CudaSpace, \ - ETI_SPEC_AVAIL) \ - KOKKOSBLAS_TRTRI_BLAS_MAGMA(Kokkos::complex, magmaDoubleComplex_ptr, \ - magma_ztrtri_gpu, LAYOUTA, Kokkos::CudaUVMSpace, \ - ETI_SPEC_AVAIL) - -#define KOKKOSBLAS_CTRTRI_BLAS(LAYOUTA, ETI_SPEC_AVAIL) \ - KOKKOSBLAS_TRTRI_BLAS_HOST(Kokkos::complex, std::complex, \ - LAYOUTA, Kokkos::HostSpace, ETI_SPEC_AVAIL) \ - KOKKOSBLAS_TRTRI_BLAS_MAGMA(Kokkos::complex, magmaFloatComplex_ptr, \ - magma_ctrtri_gpu, LAYOUTA, Kokkos::CudaSpace, \ - ETI_SPEC_AVAIL) \ - KOKKOSBLAS_TRTRI_BLAS_MAGMA(Kokkos::complex, magmaFloatComplex_ptr, \ - magma_ctrtri_gpu, LAYOUTA, Kokkos::CudaUVMSpace, \ - ETI_SPEC_AVAIL) +#ifdef KOKKOS_ENABLE_CUDA + +#define KOKKOSLAPACK_DTRTRI_LAPACK(LAYOUTA, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_HOST(double, double, LAYOUTA, Kokkos::HostSpace, \ + ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_MAGMA(double, magmaDouble_ptr, magma_dtrtri_gpu, \ + LAYOUTA, Kokkos::CudaSpace, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_MAGMA(double, magmaDouble_ptr, magma_dtrtri_gpu, \ + LAYOUTA, Kokkos::CudaUVMSpace, \ + ETI_SPEC_AVAIL) + +#define KOKKOSLAPACK_STRTRI_LAPACK(LAYOUTA, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_HOST(float, float, LAYOUTA, Kokkos::HostSpace, \ + ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_MAGMA(float, magmaFloat_ptr, magma_strtri_gpu, \ + LAYOUTA, Kokkos::CudaSpace, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_MAGMA(float, magmaFloat_ptr, magma_strtri_gpu, \ + LAYOUTA, Kokkos::CudaUVMSpace, \ + ETI_SPEC_AVAIL) + +#define KOKKOSLAPACK_ZTRTRI_LAPACK(LAYOUTA, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_HOST(Kokkos::complex, \ + std::complex, LAYOUTA, \ + Kokkos::HostSpace, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_MAGMA(Kokkos::complex, \ + magmaDoubleComplex_ptr, magma_ztrtri_gpu, \ + LAYOUTA, Kokkos::CudaSpace, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_MAGMA( \ + Kokkos::complex, magmaDoubleComplex_ptr, magma_ztrtri_gpu, \ + LAYOUTA, Kokkos::CudaUVMSpace, ETI_SPEC_AVAIL) + +#define KOKKOSLAPACK_CTRTRI_LAPACK(LAYOUTA, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_HOST(Kokkos::complex, std::complex, \ + LAYOUTA, Kokkos::HostSpace, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_MAGMA(Kokkos::complex, \ + magmaFloatComplex_ptr, magma_ctrtri_gpu, \ + LAYOUTA, Kokkos::CudaSpace, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_MAGMA( \ + Kokkos::complex, magmaFloatComplex_ptr, magma_ctrtri_gpu, \ + LAYOUTA, Kokkos::CudaUVMSpace, ETI_SPEC_AVAIL) + +#else + +#define KOKKOSLAPACK_DTRTRI_LAPACK(LAYOUTA, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_HOST(double, double, LAYOUTA, Kokkos::HostSpace, \ + ETI_SPEC_AVAIL) + +#define KOKKOSLAPACK_STRTRI_LAPACK(LAYOUTA, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_HOST(float, float, LAYOUTA, Kokkos::HostSpace, \ + ETI_SPEC_AVAIL) + +#define KOKKOSLAPACK_ZTRTRI_LAPACK(LAYOUTA, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_HOST(Kokkos::complex, \ + std::complex, LAYOUTA, \ + Kokkos::HostSpace, ETI_SPEC_AVAIL) + +#define KOKKOSLAPACK_CTRTRI_LAPACK(LAYOUTA, ETI_SPEC_AVAIL) \ + KOKKOSLAPACK_TRTRI_LAPACK_HOST(Kokkos::complex, std::complex, \ + LAYOUTA, Kokkos::HostSpace, ETI_SPEC_AVAIL) + +#endif // Handle layout permutations -KOKKOSBLAS_DTRTRI_BLAS(Kokkos::LayoutLeft, true) -KOKKOSBLAS_DTRTRI_BLAS(Kokkos::LayoutLeft, false) -KOKKOSBLAS_DTRTRI_BLAS(Kokkos::LayoutRight, true) -KOKKOSBLAS_DTRTRI_BLAS(Kokkos::LayoutRight, false) - -KOKKOSBLAS_STRTRI_BLAS(Kokkos::LayoutLeft, true) -KOKKOSBLAS_STRTRI_BLAS(Kokkos::LayoutLeft, false) -KOKKOSBLAS_STRTRI_BLAS(Kokkos::LayoutRight, true) -KOKKOSBLAS_STRTRI_BLAS(Kokkos::LayoutRight, false) - -KOKKOSBLAS_ZTRTRI_BLAS(Kokkos::LayoutLeft, true) -KOKKOSBLAS_ZTRTRI_BLAS(Kokkos::LayoutLeft, false) -KOKKOSBLAS_ZTRTRI_BLAS(Kokkos::LayoutRight, true) -KOKKOSBLAS_ZTRTRI_BLAS(Kokkos::LayoutRight, false) - -KOKKOSBLAS_CTRTRI_BLAS(Kokkos::LayoutLeft, true) -KOKKOSBLAS_CTRTRI_BLAS(Kokkos::LayoutLeft, false) -KOKKOSBLAS_CTRTRI_BLAS(Kokkos::LayoutRight, true) -KOKKOSBLAS_CTRTRI_BLAS(Kokkos::LayoutRight, false) +KOKKOSLAPACK_DTRTRI_LAPACK(Kokkos::LayoutLeft, true) +KOKKOSLAPACK_DTRTRI_LAPACK(Kokkos::LayoutLeft, false) +KOKKOSLAPACK_DTRTRI_LAPACK(Kokkos::LayoutRight, true) +KOKKOSLAPACK_DTRTRI_LAPACK(Kokkos::LayoutRight, false) + +KOKKOSLAPACK_STRTRI_LAPACK(Kokkos::LayoutLeft, true) +KOKKOSLAPACK_STRTRI_LAPACK(Kokkos::LayoutLeft, false) +KOKKOSLAPACK_STRTRI_LAPACK(Kokkos::LayoutRight, true) +KOKKOSLAPACK_STRTRI_LAPACK(Kokkos::LayoutRight, false) + +KOKKOSLAPACK_ZTRTRI_LAPACK(Kokkos::LayoutLeft, true) +KOKKOSLAPACK_ZTRTRI_LAPACK(Kokkos::LayoutLeft, false) +KOKKOSLAPACK_ZTRTRI_LAPACK(Kokkos::LayoutRight, true) +KOKKOSLAPACK_ZTRTRI_LAPACK(Kokkos::LayoutRight, false) + +KOKKOSLAPACK_CTRTRI_LAPACK(Kokkos::LayoutLeft, true) +KOKKOSLAPACK_CTRTRI_LAPACK(Kokkos::LayoutLeft, false) +KOKKOSLAPACK_CTRTRI_LAPACK(Kokkos::LayoutRight, true) +KOKKOSLAPACK_CTRTRI_LAPACK(Kokkos::LayoutRight, false) } // namespace Impl -} // nameSpace KokkosBlas +} // nameSpace KokkosLapack -#endif // KOKKOSBLAS_TRTRI_TPL_SPEC_DECL_HPP_ +#endif // KOKKOSLAPACK_TRTRI_TPL_SPEC_DECL_HPP_ diff --git a/lapack/unit_test/CMakeLists.txt b/lapack/unit_test/CMakeLists.txt new file mode 100644 index 0000000000..a2c2305a12 --- /dev/null +++ b/lapack/unit_test/CMakeLists.txt @@ -0,0 +1,94 @@ +KOKKOSKERNELS_INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/test_common) +KOKKOSKERNELS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${PACKAGE_SOURCE_DIR}/test_common) + +KOKKOSKERNELS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) +KOKKOSKERNELS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) + +##################### +# # +# Define unit-tests # +# # +##################### + +##################### +# # +# Add GPU backends # +# # +##################### +IF (KOKKOS_ENABLE_CUDA) + KOKKOSKERNELS_ADD_UNIT_TEST( + lapack_cuda + SOURCES + ${PACKAGE_SOURCE_DIR}/test_common/Test_Main.cpp + backends/Test_Cuda_Lapack.cpp + COMPONENTS lapack + ) +ENDIF () + +IF (KOKKOS_ENABLE_HIP) + KOKKOSKERNELS_ADD_UNIT_TEST( + lapack_hip + SOURCES + ${PACKAGE_SOURCE_DIR}/test_common/Test_Main.cpp + backends/Test_HIP_Lapack.cpp + COMPONENTS lapack + ) +ENDIF () + +IF (KOKKOS_ENABLE_SYCL) + KOKKOSKERNELS_ADD_UNIT_TEST( + lapack_sycl + SOURCES + ${PACKAGE_SOURCE_DIR}/test_common/Test_Main.cpp + backends/Test_SYCL_Lapack.cpp + COMPONENTS lapack + ) +ENDIF () + +IF (KOKKOS_ENABLE_OPENMPTARGET) + # KOKKOSKERNELS_ADD_UNIT_TEST( + # lapack_openmptarget + # SOURCES + # ${PACKAGE_SOURCE_DIR}/test_common/Test_Main.cpp + # backends/Test_OpenMPTarget_Lapack.cpp + # COMPONENTS lapack + # ) +ENDIF () + + + +##################### +# # +# Add CPU backends # +# # +##################### +IF (KOKKOS_ENABLE_SERIAL) + KOKKOSKERNELS_ADD_UNIT_TEST( + lapack_serial + SOURCES + ${PACKAGE_SOURCE_DIR}/test_common/Test_Main.cpp + backends/Test_Serial_Lapack.cpp + COMPONENTS lapack + ) +ENDIF () + +IF (KOKKOS_ENABLE_OPENMP) + KOKKOSKERNELS_ADD_UNIT_TEST( + lapack_openmp + SOURCES + ${PACKAGE_SOURCE_DIR}/test_common/Test_Main.cpp + backends/Test_OpenMP_Lapack.cpp + COMPONENTS lapack + ) +ENDIF () + +IF (KOKKOS_ENABLE_THREADS) + KOKKOSKERNELS_ADD_UNIT_TEST( + lapack_threads + SOURCES + ${PACKAGE_SOURCE_DIR}/test_common/Test_Main.cpp + backends/Test_Threads_Lapack.cpp + COMPONENTS lapack + ) +ENDIF () + diff --git a/lapack/unit_test/Test_Lapack.hpp b/lapack/unit_test/Test_Lapack.hpp new file mode 100644 index 0000000000..815c442884 --- /dev/null +++ b/lapack/unit_test/Test_Lapack.hpp @@ -0,0 +1,22 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER +#ifndef TEST_LAPACK_HPP +#define TEST_LAPACK_HPP + +#include "Test_Lapack_gesv.hpp" +#include "Test_Lapack_trtri.hpp" + +#endif // TEST_LAPACK_HPP diff --git a/blas/unit_test/Test_Blas_gesv.hpp b/lapack/unit_test/Test_Lapack_gesv.hpp similarity index 89% rename from blas/unit_test/Test_Blas_gesv.hpp rename to lapack/unit_test/Test_Lapack_gesv.hpp index 57ee6373bf..06f51b7eb0 100644 --- a/blas/unit_test/Test_Blas_gesv.hpp +++ b/lapack/unit_test/Test_Lapack_gesv.hpp @@ -14,19 +14,20 @@ // //@HEADER -// only enable this test where KokkosBlas supports gesv: -// CUDA+MAGMA and HOST+BLAS -#if (defined(TEST_CUDA_BLAS_CPP) && \ - defined(KOKKOSKERNELS_ENABLE_TPL_MAGMA)) || \ - (defined(KOKKOSKERNELS_ENABLE_TPL_BLAS) && \ - (defined(TEST_OPENMP_BLAS_CPP) || defined(TEST_OPENMPTARGET_BLAS_CPP) || \ - defined(TEST_SERIAL_BLAS_CPP) || defined(TEST_THREADS_BLAS_CPP))) +// only enable this test where KokkosLapack supports gesv: +// CUDA+MAGMA and HOST+LAPACK +#if (defined(TEST_CUDA_LAPACK_CPP) && \ + defined(KOKKOSKERNELS_ENABLE_TPL_MAGMA)) || \ + (defined(KOKKOSKERNELS_ENABLE_TPL_LAPACK) && \ + (defined(TEST_OPENMP_LAPACK_CPP) || \ + defined(TEST_OPENMPTARGET_LAPACK_CPP) || \ + defined(TEST_SERIAL_LAPACK_CPP) || defined(TEST_THREADS_LAPACK_CPP))) #include #include #include -#include +#include #include #include #include @@ -89,15 +90,15 @@ void impl_test_gesv(const char* mode, const char* padding, int N) { // Solve. try { - KokkosBlas::gesv(A, B, ipiv); + KokkosLapack::gesv(A, B, ipiv); } catch (const std::runtime_error& error) { // Check for expected runtime errors due to: // no-pivoting case (note: only MAGMA supports no-pivoting interface) // and no-tpl case bool nopivot_runtime_err = false; bool notpl_runtime_err = false; -#ifdef KOKKOSKERNELS_ENABLE_TPL_MAGMA // have MAGMA TPL -#ifdef KOKKOSKERNELS_ENABLE_TPL_BLAS // and have BLAS TPL +#ifdef KOKKOSKERNELS_ENABLE_TPL_MAGMA // have MAGMA TPL +#ifdef KOKKOSKERNELS_ENABLE_TPL_LAPACK // and have LAPACK TPL nopivot_runtime_err = (!std::is_same::value) && (ipiv.extent(0) == 0) && (ipiv.data() == nullptr); @@ -105,8 +106,8 @@ void impl_test_gesv(const char* mode, const char* padding, int N) { #else notpl_runtime_err = true; #endif -#else // not have MAGMA TPL -#ifdef KOKKOSKERNELS_ENABLE_TPL_BLAS // but have BLAS TPL +#else // not have MAGMA TPL +#ifdef KOKKOSKERNELS_ENABLE_TPL_LAPACK // but have LAPACK TPL nopivot_runtime_err = (ipiv.extent(0) == 0) && (ipiv.data() == nullptr); notpl_runtime_err = false; #else @@ -194,15 +195,15 @@ void impl_test_gesv_mrhs(const char* mode, const char* padding, int N, // Solve. try { - KokkosBlas::gesv(A, B, ipiv); + KokkosLapack::gesv(A, B, ipiv); } catch (const std::runtime_error& error) { // Check for expected runtime errors due to: // no-pivoting case (note: only MAGMA supports no-pivoting interface) // and no-tpl case bool nopivot_runtime_err = false; bool notpl_runtime_err = false; -#ifdef KOKKOSKERNELS_ENABLE_TPL_MAGMA // have MAGMA TPL -#ifdef KOKKOSKERNELS_ENABLE_TPL_BLAS // and have BLAS TPL +#ifdef KOKKOSKERNELS_ENABLE_TPL_MAGMA // have MAGMA TPL +#ifdef KOKKOSKERNELS_ENABLE_TPL_LAPACK // and have LAPACK TPL nopivot_runtime_err = (!std::is_same::value) && (ipiv.extent(0) == 0) && (ipiv.data() == nullptr); @@ -210,8 +211,8 @@ void impl_test_gesv_mrhs(const char* mode, const char* padding, int N, #else notpl_runtime_err = true; #endif -#else // not have MAGMA TPL -#ifdef KOKKOSKERNELS_ENABLE_TPL_BLAS // but have BLAS TPL +#else // not have MAGMA TPL +#ifdef KOKKOSKERNELS_ENABLE_TPL_LAPACK // but have LAPACK TPL nopivot_runtime_err = (ipiv.extent(0) == 0) && (ipiv.data() == nullptr); notpl_runtime_err = false; #else @@ -342,14 +343,14 @@ int test_gesv_mrhs(const char* mode) { (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) TEST_F(TestCategory, gesv_float) { - Kokkos::Profiling::pushRegion("KokkosBlas::Test::gesv_float"); + Kokkos::Profiling::pushRegion("KokkosLapack::Test::gesv_float"); test_gesv("N"); // No pivoting test_gesv("Y"); // Partial pivoting Kokkos::Profiling::popRegion(); } TEST_F(TestCategory, gesv_mrhs_float) { - Kokkos::Profiling::pushRegion("KokkosBlas::Test::gesv_mrhs_float"); + Kokkos::Profiling::pushRegion("KokkosLapack::Test::gesv_mrhs_float"); test_gesv_mrhs("N"); // No pivoting test_gesv_mrhs("Y"); // Partial pivoting Kokkos::Profiling::popRegion(); @@ -360,14 +361,14 @@ TEST_F(TestCategory, gesv_mrhs_float) { (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) TEST_F(TestCategory, gesv_double) { - Kokkos::Profiling::pushRegion("KokkosBlas::Test::gesv_double"); + Kokkos::Profiling::pushRegion("KokkosLapack::Test::gesv_double"); test_gesv("N"); // No pivoting test_gesv("Y"); // Partial pivoting Kokkos::Profiling::popRegion(); } TEST_F(TestCategory, gesv_mrhs_double) { - Kokkos::Profiling::pushRegion("KokkosBlas::Test::gesv_mrhs_double"); + Kokkos::Profiling::pushRegion("KokkosLapack::Test::gesv_mrhs_double"); test_gesv_mrhs("N"); // No pivoting test_gesv_mrhs("Y"); // Partial pivoting Kokkos::Profiling::popRegion(); @@ -378,14 +379,14 @@ TEST_F(TestCategory, gesv_mrhs_double) { (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) TEST_F(TestCategory, gesv_complex_double) { - Kokkos::Profiling::pushRegion("KokkosBlas::Test::gesv_complex_double"); + Kokkos::Profiling::pushRegion("KokkosLapack::Test::gesv_complex_double"); test_gesv, TestDevice>("N"); // No pivoting test_gesv, TestDevice>("Y"); // Partial pivoting Kokkos::Profiling::popRegion(); } TEST_F(TestCategory, gesv_mrhs_complex_double) { - Kokkos::Profiling::pushRegion("KokkosBlas::Test::gesv_mrhs_complex_double"); + Kokkos::Profiling::pushRegion("KokkosLapack::Test::gesv_mrhs_complex_double"); test_gesv_mrhs, TestDevice>("N"); // No pivoting test_gesv_mrhs, TestDevice>("Y"); // Partial pivoting Kokkos::Profiling::popRegion(); @@ -396,18 +397,18 @@ TEST_F(TestCategory, gesv_mrhs_complex_double) { (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) TEST_F(TestCategory, gesv_complex_float) { - Kokkos::Profiling::pushRegion("KokkosBlas::Test::gesv_complex_float"); + Kokkos::Profiling::pushRegion("KokkosLapack::Test::gesv_complex_float"); test_gesv, TestDevice>("N"); // No pivoting test_gesv, TestDevice>("Y"); // Partial pivoting Kokkos::Profiling::popRegion(); } TEST_F(TestCategory, gesv_mrhs_complex_float) { - Kokkos::Profiling::pushRegion("KokkosBlas::Test::gesv_mrhs_complex_float"); + Kokkos::Profiling::pushRegion("KokkosLapack::Test::gesv_mrhs_complex_float"); test_gesv_mrhs, TestDevice>("N"); // No pivoting test_gesv_mrhs, TestDevice>("Y"); // Partial pivoting Kokkos::Profiling::popRegion(); } #endif -#endif // CUDA+MAGMA or BLAS+HOST +#endif // CUDA+MAGMA or LAPACK+HOST diff --git a/blas/unit_test/Test_Blas_trtri.hpp b/lapack/unit_test/Test_Lapack_trtri.hpp similarity index 94% rename from blas/unit_test/Test_Blas_trtri.hpp rename to lapack/unit_test/Test_Lapack_trtri.hpp index aa12fa959b..a19e575d89 100644 --- a/blas/unit_test/Test_Blas_trtri.hpp +++ b/lapack/unit_test/Test_Lapack_trtri.hpp @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include @@ -118,8 +118,8 @@ int impl_test_trtri(int bad_diag_idx, const char* uplo, const char* diag, // const int As0 = A.stride(0), As1 = A.stride(1); // const int Ae0 = A.extent(0), Ae1 = A.extent(1); - // printf("KokkosBlas::trtri test for %c %c, M %d, N %d, eps %g, ViewType: %s, - // A.stride(0): %d, A.stride(1): %d, A.extent(0): %d, A.extent(1): %d + // printf("KokkosLapack::trtri test for %c %c, M %d, N %d, eps %g, ViewType: + // %s, A.stride(0): %d, A.stride(1): %d, A.extent(0): %d, A.extent(1): %d // START\n", uplo[0],diag[0],M,N,eps,typeid(ViewTypeA).name(), As0, As1, Ae0, // Ae1); fflush(stdout); @@ -141,7 +141,7 @@ int impl_test_trtri(int bad_diag_idx, const char* uplo, const char* diag, host_A(bad_diag_idx - 1, bad_diag_idx - 1) = ScalarA(0); Kokkos::deep_copy(A, host_A); } - return KokkosBlas::trtri(uplo, diag, A); + return KokkosLapack::trtri(uplo, diag, A); } // If M is greater than 100 and A is an unit triangluar matrix, make A the @@ -158,13 +158,13 @@ int impl_test_trtri(int bad_diag_idx, const char* uplo, const char* diag, using functor_type = UnitDiagTRTRI; functor_type udtrtri(A); // Initialize As diag with 1s - Kokkos::parallel_for("KokkosBlas::Test::UnitDiagTRTRI", + Kokkos::parallel_for("KokkosLapack::Test::UnitDiagTRTRI", Kokkos::RangePolicy(0, M), udtrtri); } else { //(diag[0]=='N')||(diag[0]=='n') using functor_type = NonUnitDiagTRTRI; functor_type nudtrtri(A); // Initialize As diag with A(i,i)+10 - Kokkos::parallel_for("KokkosBlas::Test::NonUnitDiagTRTRI", + Kokkos::parallel_for("KokkosLapack::Test::NonUnitDiagTRTRI", Kokkos::RangePolicy(0, M), nudtrtri); } Kokkos::fence(); @@ -195,11 +195,11 @@ int impl_test_trtri(int bad_diag_idx, const char* uplo, const char* diag, #endif // A = A^-1 - ret = KokkosBlas::trtri(uplo, diag, A); + ret = KokkosLapack::trtri(uplo, diag, A); Kokkos::fence(); if (ret) { - printf("KokkosBlas::trtri(%c, %c, %s) returned %d\n", uplo[0], diag[0], + printf("KokkosLapack::trtri(%c, %c, %s) returned %d\n", uplo[0], diag[0], typeid(ViewTypeA).name(), ret); return ret; } @@ -229,7 +229,7 @@ int impl_test_trtri(int bad_diag_idx, const char* uplo, const char* diag, vgemm.alpha = ScalarA(1); vgemm.beta = beta; Kokkos::parallel_for( - "KokkosBlas::Test::VanillaGEMM", + "KokkosLapack::Test::VanillaGEMM", Kokkos::TeamPolicy( M, Kokkos::AUTO, KokkosKernels::Impl::kk_get_max_vector_size()), @@ -362,7 +362,7 @@ int test_trtri(const char* mode) { (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) TEST_F(TestCategory, trtri_float) { - Kokkos::Profiling::pushRegion("KokkosBlas::Test::trtri_float"); + Kokkos::Profiling::pushRegion("KokkosLapack::Test::trtri_float"); test_trtri("UN"); test_trtri("UU"); test_trtri("LN"); @@ -375,7 +375,7 @@ TEST_F(TestCategory, trtri_float) { (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) TEST_F(TestCategory, trtri_double) { - Kokkos::Profiling::pushRegion("KokkosBlas::Test::trtri_double"); + Kokkos::Profiling::pushRegion("KokkosLapack::Test::trtri_double"); test_trtri("UN"); test_trtri("UU"); test_trtri("LN"); @@ -388,7 +388,7 @@ TEST_F(TestCategory, trtri_double) { (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) TEST_F(TestCategory, trtri_complex_double) { - Kokkos::Profiling::pushRegion("KokkosBlas::Test::trtri_complex_double"); + Kokkos::Profiling::pushRegion("KokkosLapack::Test::trtri_complex_double"); test_trtri, TestDevice>("UN"); test_trtri, TestDevice>("UU"); test_trtri, TestDevice>("LN"); @@ -401,7 +401,7 @@ TEST_F(TestCategory, trtri_complex_double) { (!defined(KOKKOSKERNELS_ETI_ONLY) && \ !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) TEST_F(TestCategory, trtri_complex_float) { - Kokkos::Profiling::pushRegion("KokkosBlas::Test::trtri_complex_float"); + Kokkos::Profiling::pushRegion("KokkosLapack::Test::trtri_complex_float"); test_trtri, TestDevice>("UN"); test_trtri, TestDevice>("UU"); test_trtri, TestDevice>("LN"); diff --git a/lapack/unit_test/backends/Test_Cuda_Lapack.cpp b/lapack/unit_test/backends/Test_Cuda_Lapack.cpp new file mode 100644 index 0000000000..d75988ef81 --- /dev/null +++ b/lapack/unit_test/backends/Test_Cuda_Lapack.cpp @@ -0,0 +1,22 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER +#ifndef TEST_CUDA_LAPACK_CPP +#define TEST_CUDA_LAPACK_CPP + +#include +#include + +#endif // TEST_CUDA_LAPACK_CPP diff --git a/lapack/unit_test/backends/Test_HIP_Lapack.cpp b/lapack/unit_test/backends/Test_HIP_Lapack.cpp new file mode 100644 index 0000000000..c0ec152233 --- /dev/null +++ b/lapack/unit_test/backends/Test_HIP_Lapack.cpp @@ -0,0 +1,22 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER +#ifndef TEST_HIP_LAPACK_CPP +#define TEST_HIP_LAPACK_CPP + +#include "Test_HIP.hpp" +#include "Test_Lapack.hpp" + +#endif // TEST_HIP_LAPACK_CPP diff --git a/lapack/unit_test/backends/Test_OpenMP_Lapack.cpp b/lapack/unit_test/backends/Test_OpenMP_Lapack.cpp new file mode 100644 index 0000000000..533580fd23 --- /dev/null +++ b/lapack/unit_test/backends/Test_OpenMP_Lapack.cpp @@ -0,0 +1,22 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER +#ifndef TEST_OPENMP_LAPACK_CPP +#define TEST_OPENMP_LAPACK_CPP + +#include +#include + +#endif // TEST_OPENMP_LAPACK_CPP diff --git a/lapack/unit_test/backends/Test_Serial_Lapack.cpp b/lapack/unit_test/backends/Test_Serial_Lapack.cpp new file mode 100644 index 0000000000..d0324b9642 --- /dev/null +++ b/lapack/unit_test/backends/Test_Serial_Lapack.cpp @@ -0,0 +1,22 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER +#ifndef TEST_SERIAL_LAPACK_CPP +#define TEST_SERIAL_LAPACK_CPP + +#include +#include + +#endif // TEST_SERIAL_LAPACK_CPP diff --git a/lapack/unit_test/backends/Test_Threads_Lapack.cpp b/lapack/unit_test/backends/Test_Threads_Lapack.cpp new file mode 100644 index 0000000000..aa1acbcf6c --- /dev/null +++ b/lapack/unit_test/backends/Test_Threads_Lapack.cpp @@ -0,0 +1,22 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER +#ifndef TEST_THREADS_LAPACK_CPP +#define TEST_THREADS_LAPACK_CPP + +#include +#include + +#endif // TEST_THREADS_LAPACK_CPP diff --git a/perf_test/blas/blas3/KokkosBlas_trtri_perf_test.hpp b/perf_test/blas/blas3/KokkosBlas_trtri_perf_test.hpp index cbadcef0b1..de2db8dbb0 100644 --- a/perf_test/blas/blas3/KokkosBlas_trtri_perf_test.hpp +++ b/perf_test/blas/blas3/KokkosBlas_trtri_perf_test.hpp @@ -21,7 +21,7 @@ #include -#include +#include #include "KokkosBatched_Trtri_Decl.hpp" #include "KokkosBatched_Trtri_Serial_Impl.hpp" @@ -185,7 +185,7 @@ void __do_trtri_serial_blas(options_t options, trtri_args_t trtri_args) { for (int i = 0; i < options.start.a.k; ++i) { auto A = Kokkos::subview(trtri_args.A, i, Kokkos::ALL(), Kokkos::ALL()); - KokkosBlas::trtri(&trtri_args.uplo, &trtri_args.diag, A); + KokkosLapack::trtri(&trtri_args.uplo, &trtri_args.diag, A); } // Fence after each batch operation Kokkos::fence(); @@ -196,7 +196,7 @@ void __do_trtri_serial_blas(options_t options, trtri_args_t trtri_args) { for (int i = 0; i < options.start.a.k; ++i) { auto A = Kokkos::subview(trtri_args.A, i, Kokkos::ALL(), Kokkos::ALL()); - KokkosBlas::trtri(&trtri_args.uplo, &trtri_args.diag, A); + KokkosLapack::trtri(&trtri_args.uplo, &trtri_args.diag, A); } // Fence after each batch operation Kokkos::fence(); @@ -300,7 +300,7 @@ struct parallel_blas_trtri { void operator()(const int& i) const { auto svA = Kokkos::subview(trtri_args_.A, i, Kokkos::ALL(), Kokkos::ALL()); - KokkosBlas::trtri(&trtri_args_.uplo, &trtri_args_.diag, svA); + KokkosLapack::trtri(&trtri_args_.uplo, &trtri_args_.diag, svA); } }; #endif // !KOKKOS_ENABLE_CUDA && !KOKKOS_ENABLE_HIP && diff --git a/sparse/src/KokkosSparse_sptrsv_supernode.hpp b/sparse/src/KokkosSparse_sptrsv_supernode.hpp index 0be3abac08..c6e5d406a7 100644 --- a/sparse/src/KokkosSparse_sptrsv_supernode.hpp +++ b/sparse/src/KokkosSparse_sptrsv_supernode.hpp @@ -27,7 +27,7 @@ #if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) #include "KokkosBlas3_trmm.hpp" -#include "KokkosBlas_trtri.hpp" +#include "KokkosLapack_trtri.hpp" #include "KokkosBatched_Trtri_Decl.hpp" #include "KokkosBatched_Trtri_Serial_Impl.hpp" @@ -1472,12 +1472,12 @@ void invert_supernodal_columns(KernelHandle *kernelHandle, bool unit_diag, // call trtri on device auto dViewLjj = Kokkos::subview(dViewL, range_type(0, nscol), Kokkos::ALL()); - KokkosBlas::trtri(&uplo_char, &diag_char, dViewLjj); + KokkosLapack::trtri(&uplo_char, &diag_char, dViewLjj); } else #endif { // call trtri on host - KokkosBlas::trtri(&uplo_char, &diag_char, Ljj); + KokkosLapack::trtri(&uplo_char, &diag_char, Ljj); } #ifdef KOKKOS_SPTRSV_SUPERNODE_PROFILE time1 += timer.seconds();