Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Turn on cublas and cusparse in CLANG13CUDA10 CI check #1584

Merged
merged 1 commit into from
Nov 17, 2022
Merged

Turn on cublas and cusparse in CLANG13CUDA10 CI check #1584

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions perf_test/sparse/KokkosSparse_spmv_merge.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,9 +196,8 @@ void print_help() {
}

int main(int argc, char** argv) {
using Scalar = default_scalar;
using lno_t = default_lno_t;
using size_type = default_size_type;
using Scalar = default_scalar;
using lno_t = default_lno_t;

bool compare = false;
lno_t loop = 100;
Expand Down
1 change: 0 additions & 1 deletion perf_test/sparse/KokkosSparse_sptrsv_aux.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,6 @@ bool check_cusparse(host_crsmat_t &Mtx, bool col_majorL, crsmat_t &L,

using host_values_view_t =
typename host_crsmat_t::values_type::non_const_type;
using host_scalar_t = typename host_values_view_t::value_type;

using execution_space = typename values_view_t::execution_space;
using memory_space = typename execution_space::memory_space;
Expand Down
7 changes: 6 additions & 1 deletion scripts/cm_test_all_sandia
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,11 @@ print_help() {
echo ""
echo "--make-par-level=N: Set parallelism level for builds (default: N=12)"
echo ""
echo "--with-tpls=TPLS: set KOKKOSKERNELS_ENABLE_TPLS"
echo " Provide a comma-separated list of TPLs"
echo " Valid items:"
echo " blas, mkl, cublas, cusparse, magma, armpl, rocblas, rocsparse"
echo ""

echo "ARGS: list of expressions matching compilers to test"
echo " supported compilers sems"
Expand Down Expand Up @@ -1173,7 +1178,7 @@ setup_env() {
NEW_TPL_LIST=$(echo ${NEW_TPL_LIST} | sed 's/.\w*$//')
#echo "TESTING NEW_TPL_LIST=$NEW_TPL_LIST"

KOKKOSKERNELS_ENABLE_TPL_CMD="--with-tpls=${NEW_TPL_LIST}"
KOKKOSKERNELS_ENABLE_TPL_CMD="--with-tpls=${KOKKOSKERNELS_ENABLE_TPLS},${NEW_TPL_LIST}"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure if Nathan intends to have users set tpls when doing the spotcheck or if only pre-decided tpls are actually enabled?
I would defer to him on that change to avoid breaking nightly tests, etc...

#echo "TPL USAGE: KOKKOSKERNELS_ENABLE_TPL_CMD=$KOKKOSKERNELS_ENABLE_TPL_CMD"
fi

Expand Down
35 changes: 18 additions & 17 deletions sparse/impl/KokkosSparse_spgemm_cuSPARSE_impl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,15 +66,9 @@ void cuSPARSE_symbolic(KernelHandle *handle, typename KernelHandle::nnz_lno_t m,
bin_nonzero_index_view_type entriesB, bool transposeB,
cin_row_index_view_type row_mapC) {
#ifdef KOKKOSKERNELS_ENABLE_TPL_CUSPARSE
using device1 = typename ain_row_index_view_type::device_type;
using device2 = typename ain_nonzero_index_view_type::device_type;
using idx = typename KernelHandle::nnz_lno_t;
using size_type = typename KernelHandle::size_type;
using scalar_type = typename KernelHandle::nnz_scalar_t;

// In case the KernelHandle uses const types!
using non_const_idx = typename std::remove_cv<idx>::type;
using non_const_size_type = typename std::remove_cv<size_type>::type;
using device1 = typename ain_row_index_view_type::device_type;
using device2 = typename ain_nonzero_index_view_type::device_type;
using idx = typename KernelHandle::nnz_lno_t;

// TODO this is not correct, check memory space.
if (std::is_same<Kokkos::Cuda, device1>::value) {
Expand All @@ -93,8 +87,10 @@ void cuSPARSE_symbolic(KernelHandle *handle, typename KernelHandle::nnz_lno_t m,
#if (CUDA_VERSION >= 11040)
// Newest versions of cuSPARSE have the generic SpGEMM interface, with "reuse"
// functions.
if (!std::is_same<non_const_idx, int>::value ||
!std::is_same<non_const_size_type, int>::value) {
if (!std::is_same<typename std::remove_cv<idx>::type, int>::value ||
!std::is_same<
typename std::remove_cv<typename KernelHandle::size_type>::type,
int>::value) {
throw std::runtime_error(
"cusparseSpGEMMreuse requires local ordinals to be 32-bit integer.");
}
Expand Down Expand Up @@ -174,12 +170,15 @@ void cuSPARSE_symbolic(KernelHandle *handle, typename KernelHandle::nnz_lno_t m,
(void)row_mapC;

#elif defined(CUSPARSE_VERSION) && (11000 <= CUSPARSE_VERSION)
using scalar_type = typename KernelHandle::nnz_scalar_t;
// cuSPARSE from CUDA 11.0-11.3 (inclusive) supports the new "generic" SpGEMM
// interface, just not the "reuse" set of functions. This means compute must
// be called in both symbolic and numeric (otherwise, the NNZ of C can't be
// known by symbolic)
if (!std::is_same<non_const_idx, int>::value ||
!std::is_same<non_const_size_type, int>::value) {
if (!std::is_same<typename std::remove_cv<idx>::type, int>::value ||
!std::is_same<
typename std::remove_cv<typename KernelHandle::size_type>::type,
int>::value) {
throw std::runtime_error(
"cusparseSpGEMM requires local ordinals to be 32-bit integer.");
}
Expand Down Expand Up @@ -259,15 +258,16 @@ void cuSPARSE_symbolic(KernelHandle *handle, typename KernelHandle::nnz_lno_t m,

#else

if (std::is_same<idx, int>::value && std::is_same<size_type, int>::value) {
if (std::is_same<idx, int>::value &&
std::is_same<typename KernelHandle::size_type, int>::value) {
const idx *a_xadj = (const idx *)row_mapA.data();
const idx *b_xadj = (const idx *)row_mapB.data();
idx *c_xadj = (idx *)row_mapC.data();

const idx *a_adj = entriesA.data();
const idx *b_adj = entriesB.data();
handle->create_cusparse_spgemm_handle(transposeA, transposeB);
typename KernelHandle::get_cusparse_spgemm_handle *h =
typename KernelHandle::cuSparseSpgemmHandleType *h =
handle->get_cusparse_spgemm_handle();

int nnzA = entriesA.extent(0);
Expand Down Expand Up @@ -408,8 +408,9 @@ void cuSPARSE_apply(
(void)k;

#elif (CUSPARSE_VERSION >= 11000)
const auto alpha = Kokkos::ArithTraits<scalar_type>::one();
const auto beta = Kokkos::ArithTraits<scalar_type>::zero();
using scalar_type = typename KernelHandle::nnz_scalar_t;
const auto alpha = Kokkos::ArithTraits<scalar_type>::one();
const auto beta = Kokkos::ArithTraits<scalar_type>::zero();
typename KernelHandle::cuSparseSpgemmHandleType *h =
handle->get_cusparse_spgemm_handle();
KOKKOS_CUSPARSE_SAFE_CALL(
Expand Down