diff --git a/src/common/KokkosKernels_HashmapAccumulator.hpp b/src/common/KokkosKernels_HashmapAccumulator.hpp index f9cc87e30a..6dbfca3540 100644 --- a/src/common/KokkosKernels_HashmapAccumulator.hpp +++ b/src/common/KokkosKernels_HashmapAccumulator.hpp @@ -737,7 +737,7 @@ struct HashmapAccumulator{ (*used_size_) += num_writes; }); */ - size_type my_write_index = Kokkos::atomic_fetch_add(used_size_, 1); + size_type my_write_index = Kokkos::atomic_fetch_add(used_size_, size_type(1)); if (my_write_index >= max_value_size_) { @@ -749,7 +749,7 @@ struct HashmapAccumulator{ values[my_write_index] = value; size_type hashbeginning = Kokkos::atomic_exchange(hash_begins+hash, my_write_index); if (hashbeginning == -1){ - used_hashes[Kokkos::atomic_fetch_add(used_hash_size, 1)] = hash; + used_hashes[Kokkos::atomic_fetch_add(used_hash_size, size_type(1))] = hash; } hash_nexts[my_write_index] = hashbeginning; return INSERT_SUCCESS; @@ -821,7 +821,7 @@ struct HashmapAccumulator{ if (used_size_[0] >= max_value_size_){ return INSERT_FULL; } - size_type my_write_index = Kokkos::atomic_fetch_add(used_size_, 1); + size_type my_write_index = Kokkos::atomic_fetch_add(used_size_, size_type(1)); if (my_write_index >= max_value_size_) { @@ -886,7 +886,7 @@ struct HashmapAccumulator{ size_type my_write_index = 0; if (key_not_found){ - my_write_index = Kokkos::atomic_fetch_add(used_size_, 1); + my_write_index = Kokkos::atomic_fetch_add(used_size_, size_type(1)); //my_write_index = used_size_[0]++; } else { @@ -975,7 +975,7 @@ struct HashmapAccumulator{ size_type my_write_index = 0; if (key_not_found){ - my_write_index = Kokkos::atomic_fetch_add(used_size_, 1); + my_write_index = Kokkos::atomic_fetch_add(used_size_, size_type(1)); //my_write_index = used_size_[0]++; } else { @@ -1103,7 +1103,7 @@ struct HashmapAccumulator{ //md note somehow commented part if (initial_hash_begin == end_of_link || (keys[initial_hash_begin] > key)){ { - volatile size_type my_write_index = Kokkos::atomic_fetch_add(used_size_, 1); + volatile size_type my_write_index = Kokkos::atomic_fetch_add(used_size_, size_type(1)); #if 0 if (print) std::cout << "\t 8 my_write_index :" << my_write_index << " max_value_size_:" << max_value_size_ << std::endl; #endif @@ -1255,7 +1255,7 @@ struct HashmapAccumulator{ #endif //we need to insert it to beginning. { - size_type my_write_index = Kokkos::atomic_fetch_add(used_size_, 1); + size_type my_write_index = Kokkos::atomic_fetch_add(used_size_, size_type(1)); #if 0 if (print) std::cout << "\t 13 my_write_index :" << my_write_index << " max_value_size_:" << max_value_size_ << std::endl; #endif @@ -1708,7 +1708,7 @@ struct HashmapAccumulator{ values[my_write_index] = value; size_type hashbeginning = Kokkos::atomic_exchange(hash_begins+hash, my_write_index); if (hashbeginning == -1){ - used_hashes[Kokkos::atomic_fetch_add(used_hash_size, 1)] = hash; + used_hashes[Kokkos::atomic_fetch_add(used_hash_size, size_type(1))] = hash; } hash_nexts[my_write_index] = hashbeginning; vals_counts_gmem[my_write_index] = 1; @@ -1742,7 +1742,7 @@ struct HashmapAccumulator{ return INSERT_SUCCESS; } - size_type my_write_index = Kokkos::atomic_fetch_add(used_size_,1 ); + size_type my_write_index = Kokkos::atomic_fetch_add(used_size_, size_type(1)); if (my_write_index >= max_value_size_) { return INSERT_FULL; @@ -1790,7 +1790,7 @@ struct HashmapAccumulator{ return INSERT_SUCCESS; } - size_type my_write_index = Kokkos::atomic_fetch_add(used_size_,1 ); + size_type my_write_index = Kokkos::atomic_fetch_add(used_size_, size_type(1)); if (my_write_index >= max_value_size_) { return INSERT_FULL; @@ -1877,7 +1877,7 @@ struct HashmapAccumulator{ return INSERT_SUCCESS; } - size_type my_write_index = Kokkos::atomic_fetch_add(used_size_,1 ); + size_type my_write_index = Kokkos::atomic_fetch_add(used_size_, size_type(1)); if (my_write_index >= max_value_size_) { return INSERT_FULL; @@ -1888,7 +1888,7 @@ struct HashmapAccumulator{ values[my_write_index] = value; size_type hashbeginning = Kokkos::atomic_exchange(hash_begins+hash, my_write_index); if (hashbeginning == -1){ - used_hashes[Kokkos::atomic_fetch_add(used_hash_size, 1)] = hash; + used_hashes[Kokkos::atomic_fetch_add(used_hash_size, size_type(1))] = hash; } hash_nexts[my_write_index] = hashbeginning; return INSERT_SUCCESS; @@ -1921,7 +1921,7 @@ struct HashmapAccumulator{ return INSERT_SUCCESS; } - size_type my_write_index = Kokkos::atomic_fetch_add(used_size_,1 ); + size_type my_write_index = Kokkos::atomic_fetch_add(used_size_, size_type(1)); if (my_write_index >= max_value_size_) { return INSERT_FULL; @@ -1932,7 +1932,7 @@ struct HashmapAccumulator{ //values[my_write_index] = value; size_type hashbeginning = Kokkos::atomic_exchange(hash_begins+hash, my_write_index); if (hashbeginning == -1){ - used_hashes[Kokkos::atomic_fetch_add(used_hash_size, 1)] = hash; + used_hashes[Kokkos::atomic_fetch_add(used_hash_size, size_type(1))] = hash; } hash_nexts[my_write_index] = hashbeginning; return INSERT_SUCCESS; diff --git a/src/common/KokkosKernels_PrintUtils.hpp b/src/common/KokkosKernels_PrintUtils.hpp index 1c7bf6e8dc..6d8ca517f1 100644 --- a/src/common/KokkosKernels_PrintUtils.hpp +++ b/src/common/KokkosKernels_PrintUtils.hpp @@ -63,7 +63,8 @@ struct Histogram{ KOKKOS_INLINE_FUNCTION void operator()(const size_t &ii) const { - Kokkos::atomic_fetch_add(&(outview(inview(ii))),1); + typedef typename std::remove_reference< decltype(outview(0)) >::type atomic_incr_type; + Kokkos::atomic_fetch_add(&(outview(inview(ii))), atomic_incr_type(1)); } }; diff --git a/src/common/KokkosKernels_SparseUtils.hpp b/src/common/KokkosKernels_SparseUtils.hpp index 1d2a3d6d28..86da3730de 100644 --- a/src/common/KokkosKernels_SparseUtils.hpp +++ b/src/common/KokkosKernels_SparseUtils.hpp @@ -135,7 +135,8 @@ struct TransposeMatrix{ [&] (nnz_lno_t i) { const size_type adjind = i + col_begin; const nnz_lno_t colIndex = adj[adjind]; - Kokkos::atomic_fetch_add(&(t_xadj(colIndex)),1); + typedef typename std::remove_reference< decltype( t_xadj(0) ) >::type atomic_incr_type; + Kokkos::atomic_fetch_add(&(t_xadj(colIndex)), atomic_incr_type(1)); }); }); } @@ -157,7 +158,8 @@ struct TransposeMatrix{ [&] (nnz_lno_t i) { const size_type adjind = i + col_begin; const nnz_lno_t colIndex = adj[adjind]; - const size_type pos = Kokkos::atomic_fetch_add(&(tmp_txadj(colIndex)),1); + typedef typename std::remove_reference< decltype( tmp_txadj(0) ) >::type atomic_incr_type; + const size_type pos = Kokkos::atomic_fetch_add(&(tmp_txadj(colIndex)), atomic_incr_type(1)); t_adj(pos) = row_index; if (transpose_values){ @@ -300,7 +302,8 @@ struct Fill_Reverse_Scale_Functor{ forward_type fm = forward_map[ii]; fm = fm << multiply_shift_for_scale; fm += ii >> division_shift_for_bucket; - Kokkos::atomic_fetch_add( &(reverse_map_xadj(fm)), 1); + typedef typename std::remove_reference< decltype( reverse_map_xadj(0) ) >::type atomic_incr_type; + Kokkos::atomic_fetch_add( &(reverse_map_xadj(fm)), atomic_incr_type(1)); } KOKKOS_INLINE_FUNCTION @@ -309,7 +312,8 @@ struct Fill_Reverse_Scale_Functor{ fm = fm << multiply_shift_for_scale; fm += ii >> division_shift_for_bucket; - const reverse_type future_index = Kokkos::atomic_fetch_add( &(reverse_map_xadj(fm )), 1); + typedef typename std::remove_reference< decltype( reverse_map_xadj(0) ) >::type atomic_incr_type; + const reverse_type future_index = Kokkos::atomic_fetch_add( &(reverse_map_xadj(fm) ), atomic_incr_type(1)); reverse_map_adj(future_index) = ii; } }; @@ -358,13 +362,15 @@ struct Reverse_Map_Functor{ KOKKOS_INLINE_FUNCTION void operator()(const CountTag&, const size_t &ii) const { forward_type fm = forward_map[ii]; - Kokkos::atomic_fetch_add( &(reverse_map_xadj(fm)), 1); + typedef typename std::remove_reference< decltype( reverse_map_xadj(0) ) >::type atomic_incr_type; + Kokkos::atomic_fetch_add( &(reverse_map_xadj(fm)), atomic_incr_type(1)); } KOKKOS_INLINE_FUNCTION void operator()(const FillTag&, const size_t &ii) const { forward_type c = forward_map[ii]; - const reverse_type future_index = Kokkos::atomic_fetch_add( &(reverse_map_xadj(c)), 1); + typedef typename std::remove_reference< decltype( reverse_map_xadj(0) ) >::type atomic_incr_type; + const reverse_type future_index = Kokkos::atomic_fetch_add( &(reverse_map_xadj(c)), atomic_incr_type(1)); reverse_map_adj(future_index) = ii; } }; @@ -1657,7 +1663,8 @@ void kk_create_incidence_matrix_from_lower_triangle( Kokkos::parallel_for(my_exec_space(0, ne), KOKKOS_LAMBDA(const lno_t& i) { - Kokkos::atomic_fetch_add(&(out_rowmap[in_lower_entries[i]]),1); + typedef typename std::remove_reference< decltype( out_rowmap[0] ) >::type atomic_incr_type; + Kokkos::atomic_fetch_add(&(out_rowmap[in_lower_entries[i]]), atomic_incr_type(1)); }); exec_space::fence(); @@ -1694,7 +1701,8 @@ void kk_create_incidence_matrix_from_lower_triangle( Kokkos::parallel_for(my_exec_space(0, ne), KOKKOS_LAMBDA(const size_type& edge_ind) { lno_t col = in_lower_entries[edge_ind]; - size_type write_ind = Kokkos::atomic_fetch_add(&(out_rowmap_copy(col)),1); + typedef typename std::remove_reference< decltype( out_rowmap_copy(0) ) >::type atomic_incr_type; + size_type write_ind = Kokkos::atomic_fetch_add(&(out_rowmap_copy(col)), atomic_incr_type(1)); out_entries[write_ind] = edge_ind; }); @@ -1786,8 +1794,9 @@ void kk_create_incidence_matrix_from_original_matrix( lno_t col_perm = col; if (perm) col_perm = perm[col]; if (row_perm > col_perm){ - size_type row_write_index = Kokkos::atomic_fetch_add(&(out_rowmap_copy[row]),1); - size_type col_write_index = Kokkos::atomic_fetch_add(&(out_rowmap_copy[col]),1); + typedef typename std::remove_reference< decltype( out_rowmap_copy[0] ) >::type atomic_incr_type; + size_type row_write_index = Kokkos::atomic_fetch_add(&(out_rowmap_copy[row]), atomic_incr_type(1)); + size_type col_write_index = Kokkos::atomic_fetch_add(&(out_rowmap_copy[col]), atomic_incr_type(1)); out_entries[row_write_index] = used_edge_index + used_count; out_entries[col_write_index] = used_edge_index + used_count; ++used_count; @@ -1817,8 +1826,9 @@ void kk_create_incidence_matrix_from_original_matrix( lno_t col_perm = col; if (perm) col_perm = perm[col]; if (row_perm < col_perm){ - size_type row_write_index = Kokkos::atomic_fetch_add(&(out_rowmap_copy[row]),1); - size_type col_write_index = Kokkos::atomic_fetch_add(&(out_rowmap_copy[col]),1); + typedef typename std::remove_reference< decltype( out_rowmap_copy[0] ) >::type atomic_incr_type; + size_type row_write_index = Kokkos::atomic_fetch_add(&(out_rowmap_copy[row]), atomic_incr_type(1)); + size_type col_write_index = Kokkos::atomic_fetch_add(&(out_rowmap_copy[col]), atomic_incr_type(1)); out_entries[row_write_index] = used_edge_index + used_count; out_entries[col_write_index] = used_edge_index + used_count; ++used_count; diff --git a/src/common/KokkosKernels_Utils.hpp b/src/common/KokkosKernels_Utils.hpp index e465603126..6cdeddc088 100644 --- a/src/common/KokkosKernels_Utils.hpp +++ b/src/common/KokkosKernels_Utils.hpp @@ -228,6 +228,7 @@ struct FillSymmetricEdgesHashMap{ KOKKOS_INLINE_FUNCTION void operator()(const team_member & teamMember/*, idx &nnz*/) const { + typedef typename std::remove_reference< decltype( pre_pps(0) ) >::type atomic_incr_type; idx ii = teamMember.league_rank() * teamMember.team_size()+ teamMember.team_rank(); if (ii >= num_rows) { return; @@ -244,22 +245,22 @@ struct FillSymmetricEdgesHashMap{ Kokkos::UnorderedMapInsertResult r = umap.insert(Kokkos::pair(colIndex, ii)); if (r.success()){ - Kokkos::atomic_fetch_add(&(pre_pps(ii)),1); + Kokkos::atomic_fetch_add(&(pre_pps(ii)), atomic_incr_type(1)); - Kokkos::atomic_fetch_add(&(pre_pps(colIndex)),1); + Kokkos::atomic_fetch_add(&(pre_pps(colIndex)), atomic_incr_type(1)); } } else if (colIndex > ii){ Kokkos::UnorderedMapInsertResult r = umap.insert(Kokkos::pair(ii, colIndex)); if (r.success()){ - Kokkos::atomic_fetch_add(&(pre_pps(colIndex)),1); + Kokkos::atomic_fetch_add(&(pre_pps(colIndex)), atomic_incr_type(1)); - Kokkos::atomic_fetch_add(&(pre_pps(ii)),1); + Kokkos::atomic_fetch_add(&(pre_pps(ii)), atomic_incr_type(1)); } } else { - Kokkos::atomic_fetch_add(&(pre_pps(ii)),1); + Kokkos::atomic_fetch_add(&(pre_pps(ii)), atomic_incr_type(1)); } } @@ -295,6 +296,7 @@ struct FillSymmetricLowerEdgesHashMap{ KOKKOS_INLINE_FUNCTION void operator()(const team_member & teamMember/*, idx &nnz*/) const { + typedef typename std::remove_reference< decltype( pre_pps(0) ) >::type atomic_incr_type; idx ii = teamMember.league_rank() * teamMember.team_size()+ teamMember.team_rank(); if (ii >= num_rows) { return; @@ -312,14 +314,14 @@ struct FillSymmetricLowerEdgesHashMap{ Kokkos::UnorderedMapInsertResult r = umap.insert(Kokkos::pair(colIndex, ii)); if (r.success()){ - Kokkos::atomic_fetch_add(&(pre_pps(colIndex)),1); + Kokkos::atomic_fetch_add(&(pre_pps(colIndex)), atomic_incr_type(1)); } } else if (colIndex > ii){ Kokkos::UnorderedMapInsertResult r = umap.insert(Kokkos::pair(ii, colIndex)); if (r.success()){ - Kokkos::atomic_fetch_add(&(pre_pps(ii)),1); + Kokkos::atomic_fetch_add(&(pre_pps(ii)), atomic_incr_type(1)); } } @@ -357,6 +359,7 @@ struct FillSymmetricCRS_HashMap{ KOKKOS_INLINE_FUNCTION void operator()(const team_member_t & teamMember) const { + typedef typename std::remove_reference< decltype( pre_pps(0) ) >::type atomic_incr_type; idx ii = teamMember.league_rank() * teamMember.team_size()+ teamMember.team_rank(); if (ii >= num_rows) { return; @@ -372,22 +375,22 @@ struct FillSymmetricCRS_HashMap{ if (colIndex < num_rows){ if (colIndex < ii){ if (umap.insert(Kokkos::pair(colIndex, ii)).success()){ - idx cAdjInd = Kokkos::atomic_fetch_add(&(pre_pps(colIndex)),1); - idx iAdjInd = Kokkos::atomic_fetch_add(&(pre_pps(ii)),1); + idx cAdjInd = Kokkos::atomic_fetch_add(&(pre_pps(colIndex)), atomic_incr_type(1)); + idx iAdjInd = Kokkos::atomic_fetch_add(&(pre_pps(ii)), atomic_incr_type(1)); sym_adj[cAdjInd] = ii; sym_adj[iAdjInd] = colIndex; } } else if (colIndex > ii){ if (umap.insert(Kokkos::pair(ii, colIndex)).success()){ - idx cAdjInd = Kokkos::atomic_fetch_add(&(pre_pps(colIndex)),1); - idx iAdjInd = Kokkos::atomic_fetch_add(&(pre_pps(ii)),1); + idx cAdjInd = Kokkos::atomic_fetch_add(&(pre_pps(colIndex)), atomic_incr_type(1)); + idx iAdjInd = Kokkos::atomic_fetch_add(&(pre_pps(ii)), atomic_incr_type(1)); sym_adj[cAdjInd] = ii; sym_adj[iAdjInd] = colIndex; } } else { - idx cAdjInd = Kokkos::atomic_fetch_add(&(pre_pps(colIndex)),1); + idx cAdjInd = Kokkos::atomic_fetch_add(&(pre_pps(colIndex)), atomic_incr_type(1)); sym_adj[cAdjInd] = ii; } } @@ -428,6 +431,7 @@ struct FillSymmetricEdgeList_HashMap{ KOKKOS_INLINE_FUNCTION void operator()(const team_member_t & teamMember) const { + typedef typename std::remove_reference< decltype( pps(0) ) >::type atomic_incr_type; idx ii = teamMember.league_rank() * teamMember.team_size()+ teamMember.team_rank(); if (ii >= num_rows) { return; @@ -443,14 +447,14 @@ struct FillSymmetricEdgeList_HashMap{ if (colIndex < num_rows){ if (colIndex < ii){ if (umap.insert(Kokkos::pair(colIndex, ii)).success()){ - idx cAdjInd = Kokkos::atomic_fetch_add(&(pps(colIndex)),1); + idx cAdjInd = Kokkos::atomic_fetch_add(&(pps(colIndex)), atomic_incr_type(1)); sym_src[cAdjInd] = colIndex; sym_dst[cAdjInd] = ii; } } else if (colIndex > ii){ if (umap.insert(Kokkos::pair(ii, colIndex)).success()){ - idx cAdjInd = Kokkos::atomic_fetch_add(&(pps(ii)),1); + idx cAdjInd = Kokkos::atomic_fetch_add(&(pps(ii)), atomic_incr_type(1)); sym_src[cAdjInd] = ii; sym_dst[cAdjInd] = colIndex; } @@ -485,8 +489,9 @@ struct Reverse_Map_Init{ KOKKOS_INLINE_FUNCTION void operator()(const size_t &ii) const { + typedef typename std::remove_reference< decltype( reverse_map_xadj(0) ) >::type atomic_incr_type; forward_type fm = forward_map[ii]; - Kokkos::atomic_fetch_add( &(reverse_map_xadj(fm)), 1); + Kokkos::atomic_fetch_add( &(reverse_map_xadj(fm)), atomic_incr_type(1)); } /* @@ -524,8 +529,9 @@ struct Fill_Reverse_Map{ KOKKOS_INLINE_FUNCTION void operator()(const size_t &ii) const { + typedef typename std::remove_reference< decltype( reverse_map_xadj(0) ) >::type atomic_incr_type; forward_type c = forward_map[ii]; - const reverse_type future_index = Kokkos::atomic_fetch_add( &(reverse_map_xadj(c - 1)), 1); + const reverse_type future_index = Kokkos::atomic_fetch_add( &(reverse_map_xadj(c - 1)), atomic_incr_type(1)); reverse_map_adj(future_index) = ii; } }; @@ -658,10 +664,11 @@ struct Reverse_Map_Scale_Init{ KOKKOS_INLINE_FUNCTION void operator()(const size_t &ii) const { + typedef typename std::remove_reference< decltype( reverse_map_xadj(0) ) >::type atomic_incr_type; forward_type fm = forward_map[ii]; fm = fm << multiply_shift_for_scale; fm += ii >> division_shift_for_bucket; - Kokkos::atomic_fetch_add( &(reverse_map_xadj(fm)), 1); + Kokkos::atomic_fetch_add( &(reverse_map_xadj(fm)), atomic_incr_type(1)); } }; @@ -691,11 +698,12 @@ struct Fill_Reverse_Scale_Map{ KOKKOS_INLINE_FUNCTION void operator()(const size_t &ii) const { + typedef typename std::remove_reference< decltype( reverse_map_xadj(0) ) >::type atomic_incr_type; forward_type fm = forward_map[ii]; fm = fm << multiply_shift_for_scale; fm += ii >> division_shift_for_bucket; - const reverse_type future_index = Kokkos::atomic_fetch_add( &(reverse_map_xadj(fm - 1)), 1); + const reverse_type future_index = Kokkos::atomic_fetch_add( &(reverse_map_xadj(fm - 1)), atomic_incr_type(1)); reverse_map_adj(future_index) = ii; } }; @@ -1482,6 +1490,7 @@ struct TransposeMatrix2{ KOKKOS_INLINE_FUNCTION void operator()(const CountTag&, const team_count_member_t & teamMember) const { + typedef typename std::remove_reference< decltype( t_xadj(0) ) >::type atomic_incr_type; const nnz_lno_t row_index = teamMember.league_rank() * teamMember.team_size() + teamMember.team_rank(); @@ -1495,12 +1504,13 @@ struct TransposeMatrix2{ [&] (nnz_lno_t i) { const size_type adjind = i + col_begin; const nnz_lno_t colIndex = adj[adjind]; - Kokkos::atomic_fetch_add(&(t_xadj(colIndex)),1); + Kokkos::atomic_fetch_add(&(t_xadj(colIndex)), atomic_incr_type(1)); }); } KOKKOS_INLINE_FUNCTION void operator()(const FillTag&, const team_fill_member_t & teamMember) const { + typedef typename std::remove_reference< decltype( tmp_txadj(0) ) >::type atomic_incr_type; const nnz_lno_t row_index = teamMember.league_rank() * teamMember.team_size() + teamMember.team_rank(); if (row_index >= num_rows) return; @@ -1512,7 +1522,7 @@ struct TransposeMatrix2{ [&] (nnz_lno_t i) { const size_type adjind = i + col_begin; const nnz_lno_t colIndex = adj[adjind]; - const size_type pos = Kokkos::atomic_fetch_add(&(tmp_txadj(colIndex)),1); + const size_type pos = Kokkos::atomic_fetch_add(&(tmp_txadj(colIndex)), atomic_incr_type(1)); t_adj(pos) = row_index; if (transpose_values){ diff --git a/src/graph/KokkosGraph_GraphColorHandle.hpp b/src/graph/KokkosGraph_GraphColorHandle.hpp index 2b2e4ab549..ca3a559fa5 100644 --- a/src/graph/KokkosGraph_GraphColorHandle.hpp +++ b/src/graph/KokkosGraph_GraphColorHandle.hpp @@ -372,7 +372,7 @@ class GraphColoringHandle KOKKOS_INLINE_FUNCTION void operator()(const team_member_t & teamMember) const { - + typedef typename std::remove_reference< decltype( lower_xadj_counts(0) ) >::type atomic_incr_type; nnz_lno_t ii = teamMember.league_rank() * teamMember.team_size()+ teamMember.team_rank(); if (ii >= nv) { @@ -390,7 +390,7 @@ class GraphColoringHandle nnz_lno_t n = adj[adjind]; if (ii < n && n < nv){ size_type position = - Kokkos::atomic_fetch_add( &(lower_xadj_counts(ii)), 1); + Kokkos::atomic_fetch_add( &(lower_xadj_counts(ii)), atomic_incr_type(1)); lower_srcs(position) = ii; lower_dsts(position) = n; } diff --git a/src/graph/impl/KokkosGraph_Distance2Color_impl.hpp b/src/graph/impl/KokkosGraph_Distance2Color_impl.hpp index 1dfd6d45d9..73ad885c18 100644 --- a/src/graph/impl/KokkosGraph_Distance2Color_impl.hpp +++ b/src/graph/impl/KokkosGraph_Distance2Color_impl.hpp @@ -845,6 +845,7 @@ class GraphColorD2 KOKKOS_INLINE_FUNCTION void operator()(const nnz_lno_t vid_, nnz_lno_t& numConflicts) const { + typedef typename std::remove_reference< decltype( _recolorListLength() ) >::type atomic_incr_type; nnz_lno_t vid = _vertexList(vid_); color_t my_color = _colors(vid); @@ -865,7 +866,7 @@ class GraphColorD2 { _colors(vid) = 0; // uncolor vertex // Atomically add vertex to recolorList - const nnz_lno_t k = Kokkos::atomic_fetch_add( &_recolorListLength(), 1); + const nnz_lno_t k = Kokkos::atomic_fetch_add( &_recolorListLength(), atomic_incr_type(1)); _recolorList(k) = vid; numConflicts += 1; break; // Can exit if vertex gets marked as a conflict. diff --git a/src/graph/impl/KokkosGraph_GraphColor_impl.hpp b/src/graph/impl/KokkosGraph_GraphColor_impl.hpp index 9f0249ec97..c8d066d066 100644 --- a/src/graph/impl/KokkosGraph_GraphColor_impl.hpp +++ b/src/graph/impl/KokkosGraph_GraphColor_impl.hpp @@ -2026,6 +2026,7 @@ class GraphColor_VB:public GraphColor ::type atomic_incr_type; nnz_lno_t i = _vertexList(ii); color_t my_color = _colors(i); @@ -2050,7 +2051,7 @@ class GraphColor_VB:public GraphColor ::type atomic_incr_type; nnz_lno_t i = _vertexList(ii); color_t my_color = _colors(i); if (my_color == 0){ // this should only happen when one_color_set_per_iteration is set to true. - const nnz_lno_t k = Kokkos::atomic_fetch_add( &_recolorListLength(), 1); + const nnz_lno_t k = Kokkos::atomic_fetch_add( &_recolorListLength(), atomic_incr_type(1)); _recolorList(k) = i; numConflicts++; } @@ -2257,7 +2259,7 @@ class GraphColor_VB:public GraphColor ::type atomic_incr_type; size_type w = _edge_conflict_indices(ii); if(_edge_conflict_marker(w)){ - const size_type future_index = Kokkos::atomic_fetch_add( &_new_index(), 1); + const size_type future_index = Kokkos::atomic_fetch_add( &_new_index(), atomic_incr_type(1)); _new_edge_conflict_indices(future_index) = w; } } diff --git a/src/sparse/impl/KokkosSparse_spgemm_impl_kkmem.hpp b/src/sparse/impl/KokkosSparse_spgemm_impl_kkmem.hpp index 8e77d7d0c4..f1aefe2add 100644 --- a/src/sparse/impl/KokkosSparse_spgemm_impl_kkmem.hpp +++ b/src/sparse/impl/KokkosSparse_spgemm_impl_kkmem.hpp @@ -619,6 +619,7 @@ struct KokkosSPGEMM int thread_rank = teamMember.team_rank(); int vector_rank = 0; + typedef typename std::remove_reference< decltype( *used_hash_sizes ) >::type atomic_incr_type; Kokkos::parallel_scan( Kokkos::ThreadVectorRange(teamMember, vector_size), [&] (const int threadid, int &update, const bool final) { @@ -870,7 +871,7 @@ struct KokkosSPGEMM } if (fail){ nnz_lno_t write_index = 0; - write_index = Kokkos::atomic_fetch_add(used_hash_sizes + 1, 1); + write_index = Kokkos::atomic_fetch_add(used_hash_sizes + 1, atomic_incr_type(1)); c_row[write_index] = my_b_col; c_row_vals[write_index] = my_b_val; } @@ -890,7 +891,7 @@ struct KokkosSPGEMM if (my_key != init_value){ scalar_t my_val = vals[my_index]; nnz_lno_t write_index = 0; - write_index = Kokkos::atomic_fetch_add(used_hash_sizes + 1, 1); + write_index = Kokkos::atomic_fetch_add(used_hash_sizes + 1, atomic_incr_type(1)); c_row[write_index] = my_key; c_row_vals[write_index] = my_val; } @@ -923,6 +924,7 @@ struct KokkosSPGEMM int thread_rank = teamMember.team_rank(); int vector_rank = 0; + typedef typename std::remove_reference< decltype( *used_hash_sizes ) >::type atomic_incr_type; Kokkos::parallel_scan( Kokkos::ThreadVectorRange(teamMember, vector_size), [&] (const int threadid, int &update, const bool final) { @@ -1084,7 +1086,7 @@ struct KokkosSPGEMM nnz_lno_t my_key = keys[my_index]; if (my_key != init_value){ scalar_t my_val = vals[my_index]; - nnz_lno_t write_index = Kokkos::atomic_fetch_add(used_hash_sizes, 1); + nnz_lno_t write_index = Kokkos::atomic_fetch_add(used_hash_sizes, atomic_incr_type(1)); c_row[write_index] = my_key; c_row_vals[write_index] = my_val; } diff --git a/src/sparse/impl/KokkosSparse_spgemm_impl_symbolic.hpp b/src/sparse/impl/KokkosSparse_spgemm_impl_symbolic.hpp index 399ca91489..b09369a603 100644 --- a/src/sparse/impl/KokkosSparse_spgemm_impl_symbolic.hpp +++ b/src/sparse/impl/KokkosSparse_spgemm_impl_symbolic.hpp @@ -2693,6 +2693,8 @@ struct KokkosSPGEMM //used_hash_sizes hold the size of 1st and 2nd level hashes volatile nnz_lno_t *used_hash_sizes = (volatile nnz_lno_t *) (all_shared_memory); + typedef typename std::remove_reference< decltype( *used_hash_sizes ) >::type atomic_incr_type; + all_shared_memory += sizeof(nnz_lno_t) * 2; nnz_lno_t *globally_used_hash_count = (nnz_lno_t *) (all_shared_memory); @@ -2847,7 +2849,7 @@ struct KokkosSPGEMM while (c_rows){ if (c_rows & unit){ - size_type wind = Kokkos::atomic_fetch_add(used_hash_sizes, 1); + size_type wind = Kokkos::atomic_fetch_add(used_hash_sizes, atomic_incr_type(1)); entriesSetIndicesC(wind + row_begin) = set_size * c_rows_setind + current_row; } current_row++; @@ -2871,7 +2873,7 @@ struct KokkosSPGEMM while (c_rows){ if (c_rows & unit){ - size_type wind = Kokkos::atomic_fetch_add(used_hash_sizes, 1); + size_type wind = Kokkos::atomic_fetch_add(used_hash_sizes, atomic_incr_type(1)); entriesSetIndicesC(wind + row_begin) = set_size * c_rows_setind + current_row; } current_row++; diff --git a/src/sparse/impl/KokkosSparse_spgemm_impl_triangle.hpp b/src/sparse/impl/KokkosSparse_spgemm_impl_triangle.hpp index 627374d0a6..d91e035558 100644 --- a/src/sparse/impl/KokkosSparse_spgemm_impl_triangle.hpp +++ b/src/sparse/impl/KokkosSparse_spgemm_impl_triangle.hpp @@ -1318,6 +1318,7 @@ struct KokkosSPGEMM //used_hash_sizes hold the size of 1st and 2nd level hashes volatile nnz_lno_t *used_hash_sizes = (volatile nnz_lno_t *) (all_shared_memory); + typedef typename std::remove_reference< decltype( *used_hash_sizes ) >::type atomic_incr_type; all_shared_memory += sizeof(nnz_lno_t) * 2; nnz_lno_t *globally_used_hash_count = (nnz_lno_t *) (all_shared_memory); @@ -1569,7 +1570,7 @@ struct KokkosSPGEMM while (c_rows){ if (c_rows & unit){ - size_type wind = Kokkos::atomic_fetch_add(used_hash_sizes, 1); + size_type wind = Kokkos::atomic_fetch_add(used_hash_sizes, atomic_incr_type(1)); entriesC[wind + row_begin] = set_size * c_rows_setind + current_row; } current_row++; @@ -1592,7 +1593,7 @@ struct KokkosSPGEMM while (c_rows){ if (c_rows & unit){ - size_type wind = Kokkos::atomic_fetch_add(used_hash_sizes, 1); + size_type wind = Kokkos::atomic_fetch_add(used_hash_sizes, atomic_incr_type(1)); entriesC[wind + row_begin] = set_size * c_rows_setind + current_row; } current_row++; diff --git a/src/sparse/impl/KokkosSparse_spgemm_impl_triangle_no_compression.hpp b/src/sparse/impl/KokkosSparse_spgemm_impl_triangle_no_compression.hpp index 30dfc9a5c5..a1259a167c 100644 --- a/src/sparse/impl/KokkosSparse_spgemm_impl_triangle_no_compression.hpp +++ b/src/sparse/impl/KokkosSparse_spgemm_impl_triangle_no_compression.hpp @@ -881,6 +881,7 @@ struct KokkosSPGEMM //used_hash_sizes hold the size of 1st and 2nd level hashes volatile nnz_lno_t *used_hash_sizes = (volatile nnz_lno_t *) (all_shared_memory); + typedef typename std::remove_reference< decltype( *used_hash_sizes ) >::type atomic_incr_type; all_shared_memory += sizeof(nnz_lno_t) * 2; nnz_lno_t *globally_used_hash_count = (nnz_lno_t *) (all_shared_memory); @@ -1132,7 +1133,7 @@ struct KokkosSPGEMM while (c_rows){ if (c_rows & unit){ - size_type wind = Kokkos::atomic_fetch_add(used_hash_sizes, 1); + size_type wind = Kokkos::atomic_fetch_add(used_hash_sizes, atomic_incr_type(1)); entriesC[wind + row_begin] = set_size * c_rows_setind + current_row; } current_row++; @@ -1155,7 +1156,7 @@ struct KokkosSPGEMM while (c_rows){ if (c_rows & unit){ - size_type wind = Kokkos::atomic_fetch_add(used_hash_sizes, 1); + size_type wind = Kokkos::atomic_fetch_add(used_hash_sizes, atomic_incr_type(1)); entriesC[wind + row_begin] = set_size * c_rows_setind + current_row; } current_row++;