diff --git a/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp b/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp
index 1e5c098814..f4254f129d 100644
--- a/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp
+++ b/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp
@@ -45,7 +45,7 @@
 #ifndef KOKKOSSPARSE_IMPL_SPMV_TENSOR_CORE_DEF_HPP_
 #define KOKKOSSPARSE_IMPL_SPMV_TENSOR_CORE_DEF_HPP_
 
-#if defined(KOKKOS_ENABLE_CUDA) && \
+#if defined(KOKKOS_ENABLE_CUDA) && defined(__CUDA_ARCH__) && \
     (defined(KOKKOS_ARCH_VOLTA) || defined(KOKKOS_ARCH_AMPERE))
 
 #include <type_traits>
diff --git a/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp b/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp
index 621c52c157..10abd991d1 100644
--- a/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp
+++ b/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_spec.hpp
@@ -50,7 +50,8 @@
 
 #include "KokkosSparse_BsrMatrix.hpp"
 #include "KokkosKernels_Controls.hpp"
-#if !defined(KOKKOSKERNELS_ETI_ONLY) || KOKKOSKERNELS_IMPL_COMPILE_LIBRARY
+#if !defined(KOKKOSKERNELS_ETI_ONLY) || \
+    KOKKOSKERNELS_IMPL_COMPILE_LIBRARY && defined(__CUDA_ARCH__)
 #include <KokkosSparse_spmv_bsrmatrix_impl.hpp>
 #endif
 
@@ -136,7 +137,9 @@ struct SPMV_BSRMATRIX<AT, AO, AD, AM, AS, XT, XL, XD, XM, YT, YL, YD, YM,
       }
     }
 
-#if defined(KOKKOS_ARCH_AMPERE)
+    // If not compiling with the cuda compiler, no need to try
+    // to compile the wmma functions...
+#if defined(KOKKOS_ARCH_AMPERE) && defined(__CUDA_ARCH__)
     typedef typename XVector::non_const_value_type XScalar;
     typedef typename AMatrix::non_const_value_type AScalar;
     typedef Kokkos::Experimental::half_t Half;
@@ -173,7 +176,7 @@ struct SPMV_BSRMATRIX<AT, AO, AD, AM, AS, XT, XL, XD, XM, YT, YL, YD, YM,
                                         4>::dispatch(alpha, A, x, beta, y);
     }
 
-#elif defined(KOKKOS_ARCH_VOLTA)
+#elif defined(KOKKOS_ARCH_VOLTA) && defined(__CUDA_ARCH__)
     /* Volta has float += half * half
        use it for all matrices
     */