Merge branch 'branch-21.12' of github.com:rapidsai/cugraph into fea_e…

…xt_binding_rw_biased

CHANGELOG.md

@@ -1,3 +1,7 @@
+# cuGraph 21.12.00 (Date TBD)
+
+Please see https://github.com/rapidsai/cugraph/releases/tag/v21.12.00a for the latest changes to this development branch.
+
 # cuGraph 21.10.00 (Date TBD)
 
 Please see https://github.com/rapidsai/cugraph/releases/tag/v21.10.00a for the latest changes to this development branch.

benchmarks/python_e2e/main.py

@@ -110,7 +110,7 @@ def run(algos,
         funcs.teardown(*setup_objs)
         log("done.")
 
-    return success is True
+    return 0 if success else 1
 
 
 if __name__ == "__main__":

conda/environments/cugraph_dev_cuda11.0.yml

@@ -6,14 +6,14 @@ channels:
 - conda-forge
 dependencies:
 - cudatoolkit=11.0
-- cudf=21.10.*
-- libcudf=21.10.*
-- rmm=21.10.*
-- librmm=21.10.*
-- dask>=2021.6.0
-- distributed>=2021.6.0
-- dask-cuda=21.10.*
-- dask-cudf=21.10.*
+- cudf=21.12.*
+- libcudf=21.12.*
+- rmm=21.12.*
+- librmm=21.12.*
+- dask=2021.09.1
+- distributed=2021.09.1
+- dask-cuda=21.12.*
+- dask-cudf=21.12.*
 - nccl>=2.9.9
 - ucx-py=0.22.*
 - ucx-proc=*=gpu

conda/environments/cugraph_dev_cuda11.2.yml

@@ -6,14 +6,14 @@ channels:
 - conda-forge
 dependencies:
 - cudatoolkit=11.2
-- cudf=21.10.*
-- libcudf=21.10.*
-- rmm=21.10.*
-- librmm=21.10.*
-- dask>=2021.6.0
-- distributed>=2021.6.0
-- dask-cuda=21.10.*
-- dask-cudf=21.10.*
+- cudf=21.12.*
+- libcudf=21.12.*
+- rmm=21.12.*
+- librmm=21.12.*
+- dask=2021.09.1
+- distributed=2021.09.1
+- dask-cuda=21.12.*
+- dask-cudf=21.12.*
 - nccl>=2.9.9
 - ucx-py=0.22.*
 - ucx-proc=*=gpu

conda/environments/cugraph_dev_cuda11.4.yml

@@ -6,14 +6,14 @@ channels:
 - conda-forge
 dependencies:
 - cudatoolkit=11.4
-- cudf=21.10.*
-- libcudf=21.10.*
-- rmm=21.10.*
-- librmm=21.10.*
-- dask>=2021.6.0
-- distributed>=2021.6.0
-- dask-cuda=21.10.*
-- dask-cudf=21.10.*
+- cudf=21.12.*
+- libcudf=21.12.*
+- rmm=21.12.*
+- librmm=21.12.*
+- dask=2021.09.1
+- distributed=2021.09.1
+- dask-cuda=21.12.*
+- dask-cudf=21.12.*
 - nccl>=2.9.9
 - ucx-py=0.22.*
 - ucx-proc=*=gpu

conda/recipes/cugraph/meta.yaml

@@ -37,8 +37,8 @@ requirements:
     - cudf={{ minor_version }}
     - dask-cudf {{ minor_version }}
     - dask-cuda {{ minor_version }}
-    - dask>=2021.6.0
-    - distributed>=2021.6.0
+    - dask=2021.09.1
+    - distributed=2021.09.1
     - ucx-py 0.22
     - ucx-proc=*=gpu
     - {{ pin_compatible('cudatoolkit', max_pin='x.x') }}

cpp/CMakeLists.txt

@@ -27,7 +27,17 @@ include(rapids-find)
 
 rapids_cuda_init_architectures(CUGRAPH)
 
-project(CUGRAPH VERSION 21.10.00 LANGUAGES C CXX CUDA)
+project(CUGRAPH VERSION 21.12.00 LANGUAGES C CXX CUDA)
+
+if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA" AND
+   CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.0)
+    message(FATAL_ERROR "CUDA compiler version must be at least 11.0")
+endif()
+
+if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND
+   CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.3)
+    message(FATAL_ERROR "GCC compiler must be at least 9.3")
+endif()
 
 # Remove the following archs from CMAKE_CUDA_ARCHITECTURES that
 # cuhornet currently doesn't support

cpp/cmake/thirdparty/get_cuco.cmake

@@ -20,7 +20,7 @@ function(find_and_configure_cuco VERSION)
       GLOBAL_TARGETS cuco::cuco
       CPM_ARGS
         GIT_REPOSITORY https://github.com/NVIDIA/cuCollections.git
-        GIT_TAG        b1fea0cbe4c384160740af00f7c8760846539abb
+        GIT_TAG        729857a5698a0e8d8f812e0464f65f37854ae17b
         OPTIONS        "BUILD_TESTS OFF"
                        "BUILD_BENCHMARKS OFF"
                        "BUILD_EXAMPLES OFF"

cpp/include/cugraph/graph.hpp

@@ -51,6 +51,9 @@ struct graph_meta_t<vertex_t, edge_t, multi_gpu, std::enable_if_t<multi_gpu>> {
 
   // segment offsets based on vertex degree, relevant only if vertex IDs are renumbered
   std::optional<std::vector<vertex_t>> segment_offsets{std::nullopt};
+
+  vertex_t num_local_unique_edge_rows{};
+  vertex_t num_local_unique_edge_cols{};
 };
 
 // single-GPU version
@@ -136,6 +139,22 @@ class graph_t<vertex_t, edge_t, weight_t, store_transposed, multi_gpu, std::enab
         this->get_graph_properties(),
         partition_,
         adj_matrix_partition_segment_offsets_,
+        local_sorted_unique_edge_rows_
+          ? std::optional<vertex_t const*>{(*local_sorted_unique_edge_rows_).data()}
+          : std::nullopt,
+        local_sorted_unique_edge_rows_
+          ? std::optional<vertex_t const*>{(*local_sorted_unique_edge_rows_).data() +
+                                           (*local_sorted_unique_edge_rows_).size()}
+          : std::nullopt,
+        local_sorted_unique_edge_row_offsets_,
+        local_sorted_unique_edge_cols_
+          ? std::optional<vertex_t const*>{(*local_sorted_unique_edge_cols_).data()}
+          : std::nullopt,
+        local_sorted_unique_edge_cols_
+          ? std::optional<vertex_t const*>{(*local_sorted_unique_edge_cols_).data() +
+                                           (*local_sorted_unique_edge_cols_).size()}
+          : std::nullopt,
+        local_sorted_unique_edge_col_offsets_,
       },
       false);
   }
@@ -155,6 +174,13 @@ class graph_t<vertex_t, edge_t, weight_t, store_transposed, multi_gpu, std::enab
   // segment offsets within the vertex partition based on vertex degree, relevant only if
   // segment_offsets.size() > 0
   std::optional<std::vector<vertex_t>> adj_matrix_partition_segment_offsets_{std::nullopt};
+
+  // if valid, store row/column properties in key/value pairs (this saves memory if # unique edge
+  // rows/cols << V / row_comm_size|col_comm_size).
+  std::optional<rmm::device_uvector<vertex_t>> local_sorted_unique_edge_rows_{std::nullopt};
+  std::optional<rmm::device_uvector<vertex_t>> local_sorted_unique_edge_cols_{std::nullopt};
+  std::optional<std::vector<vertex_t>> local_sorted_unique_edge_row_offsets_{std::nullopt};
+  std::optional<std::vector<vertex_t>> local_sorted_unique_edge_col_offsets_{std::nullopt};
 };
 
 // single-GPU version

cpp/include/cugraph/graph_functions.hpp

@@ -37,6 +37,9 @@ struct renumber_meta_t<vertex_t, edge_t, multi_gpu, std::enable_if_t<multi_gpu>>
   edge_t number_of_edges{};
   partition_t<vertex_t> partition{};
   std::vector<vertex_t> segment_offsets{};
+
+  vertex_t num_local_unique_edge_majors{};
+  vertex_t num_local_unique_edge_minors{};
 };
 
 template <typename vertex_t, typename edge_t, bool multi_gpu>
@@ -88,9 +91,9 @@ struct renumber_meta_t<vertex_t, edge_t, multi_gpu, std::enable_if_t<!multi_gpu>
  * Tuple of labels (vertex IDs before renumbering) for the entire set of vertices (assigned to this
  * process in multi-GPU) and meta-data collected while renumbering. The meta-data includes total
  * number of vertices, total number of edges, partition_t object storing graph partitioning
- * information,  and vertex partition segment offsets (a vertex partition is partitioned to multiple
- * segments based on vertex degrees). This meta-data is expected to be used in graph construction &
- * graph primitives.
+ * information, vertex partition segment offsets (a vertex partition is partitioned to multiple
+ * segments based on vertex degrees), and the number of local unique edge major & minor vertex IDs.
+ * This meta-data is expected to be used in graph construction & graph primitives.
  */
 template <typename vertex_t, typename edge_t, bool multi_gpu>
 std::enable_if_t<

cpp/include/cugraph/graph_view.hpp

@@ -223,6 +223,11 @@ namespace detail {
 
 using namespace cugraph::visitors;
 
+// FIXME: threshold values require tuning (currently disabled)
+// use (key, value) pairs to store row/column properties if (unique edge rows/cols) over (V /
+// row_comm_size|col_comm_size) is smaller than the threshold value
+double constexpr row_col_properties_kv_pair_fill_ratio_threshold = 0.0;
+
 // FIXME: threshold values require tuning
 // use the hypersparse format (currently, DCSR or DCSC) for the vertices with their degrees smaller
 // than col_comm_size * hypersparse_threshold_ratio, should be less than 1.0
@@ -300,6 +305,13 @@ struct graph_view_meta_t<vertex_t, edge_t, multi_gpu, std::enable_if_t<multi_gpu
 
   // segment offsets based on vertex degree, relevant only if vertex IDs are renumbered
   std::optional<std::vector<vertex_t>> adj_matrix_partition_segment_offsets{};
+
+  std::optional<vertex_t const*> local_sorted_unique_edge_row_first{std::nullopt};
+  std::optional<vertex_t const*> local_sorted_unique_edge_row_last{std::nullopt};
+  std::optional<std::vector<vertex_t>> local_sorted_unique_edge_row_offsets{std::nullopt};
+  std::optional<vertex_t const*> local_sorted_unique_edge_col_first{std::nullopt};
+  std::optional<vertex_t const*> local_sorted_unique_edge_col_last{std::nullopt};
+  std::optional<std::vector<vertex_t>> local_sorted_unique_edge_col_offsets{std::nullopt};
 };
 
 // single-GPU version
@@ -587,6 +599,11 @@ class graph_view_t<vertex_t,
     return local_sorted_unique_edge_row_last_;
   }
 
+  std::optional<std::vector<vertex_t>> get_local_sorted_unique_edge_row_offsets() const
+  {
+    return local_sorted_unique_edge_row_offsets_;
+  }
+
   std::optional<vertex_t const*> get_local_sorted_unique_edge_col_begin() const
   {
     return local_sorted_unique_edge_col_first_;
@@ -597,6 +614,11 @@ class graph_view_t<vertex_t,
     return local_sorted_unique_edge_col_last_;
   }
 
+  std::optional<std::vector<vertex_t>> get_local_sorted_unique_edge_col_offsets() const
+  {
+    return local_sorted_unique_edge_col_offsets_;
+  }
+
  private:
   std::vector<edge_t const*> adj_matrix_partition_offsets_{};
   std::vector<vertex_t const*> adj_matrix_partition_indices_{};
@@ -613,11 +635,14 @@ class graph_view_t<vertex_t,
   // segment offsets based on vertex degree, relevant only if vertex IDs are renumbered
   std::optional<std::vector<vertex_t>> adj_matrix_partition_segment_offsets_{};
 
-  // FIXME: to be implemented.
+  // if valid, store row/column properties in key/value pairs (this saves memory if # unique edge
+  // rows/cols << V / row_comm_size|col_comm_size).
   std::optional<vertex_t const*> local_sorted_unique_edge_row_first_{std::nullopt};
   std::optional<vertex_t const*> local_sorted_unique_edge_row_last_{std::nullopt};
+  std::optional<std::vector<vertex_t>> local_sorted_unique_edge_row_offsets_{std::nullopt};
   std::optional<vertex_t const*> local_sorted_unique_edge_col_first_{std::nullopt};
   std::optional<vertex_t const*> local_sorted_unique_edge_col_last_{std::nullopt};
+  std::optional<std::vector<vertex_t>> local_sorted_unique_edge_col_offsets_{std::nullopt};
 };
 
 // single-GPU version