Merge f3e27fa into b89eb18

ci/gpu/build.sh

@@ -88,7 +88,7 @@ conda list --show-channel-urls
 
 if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     gpuci_logger "Build from source"
-    $WORKSPACE/build.sh -v clean libcugraph cugraph
+    $WORKSPACE/build.sh -v clean libcugraph pylibcugraph cugraph
 else
     export LIBCUGRAPH_BUILD_DIR="$WORKSPACE/ci/artifacts/cugraph/cpu/conda_work/cpp/build"
 
@@ -108,8 +108,8 @@ else
     pip install "git+https://github.com/dask/distributed.git" --upgrade --no-deps
     pip install "git+https://github.com/dask/dask.git" --upgrade --no-deps
 
-    echo "Build cugraph..."
-    $WORKSPACE/build.sh cugraph
+    echo "Build pylibcugraph and cugraph..."
+    $WORKSPACE/build.sh pylibcugraph cugraph
 fi
 
 ################################################################################

ci/test.sh

@@ -26,7 +26,7 @@ GTEST_ARGS="--gtest_output=xml:${CUGRAPH_ROOT}/test-results/"
 DOWNLOAD_MODE=""
 EXITCODE=0
 
-export RAPIDS_DATASET_ROOT_DIR=${CUGRAPH_ROOT}/datasets
+export RAPIDS_DATASET_ROOT_DIR=${RAPIDS_DATASET_ROOT_DIR:-${CUGRAPH_ROOT}/datasets}
 
 # FIXME: consider using getopts for option parsing
 function hasArg {
@@ -78,9 +78,14 @@ else
     done
 fi
 
+echo "Python pytest for pylibcugraph..."
+cd ${CUGRAPH_ROOT}/python/pylibcugraph/pylibcugraph
+pytest --cache-clear --junitxml=${CUGRAPH_ROOT}/junit-cugraph.xml -v --cov-config=.coveragerc --cov=pylibcugraph --cov-report=xml:${WORKSPACE}/python/pylibcugraph/pylibcugraph-coverage.xml --cov-report term --ignore=raft --benchmark-disable
+echo "Ran Python pytest for pylibcugraph : return code was: $?, test script exit code is now: $EXITCODE"
+
 echo "Python pytest for cuGraph..."
-cd ${CUGRAPH_ROOT}/python/cugraph
-pytest --cache-clear --junitxml=${CUGRAPH_ROOT}/junit-cugraph.xml -v --cov-config=.coveragerc --cov=cugraph --cov-report=xml:${WORKSPACE}/python/cugraph/cugraph-coverage.xml --cov-report term --ignore=cugraph/raft --benchmark-disable
+cd ${CUGRAPH_ROOT}/python/cugraph/cugraph
+pytest --cache-clear --junitxml=${CUGRAPH_ROOT}/junit-cugraph.xml -v --cov-config=.coveragerc --cov=cugraph --cov-report=xml:${WORKSPACE}/python/cugraph/cugraph-coverage.xml --cov-report term --ignore=raft --benchmark-disable
 echo "Ran Python pytest for cugraph : return code was: $?, test script exit code is now: $EXITCODE"
 
 echo "Python benchmarks for cuGraph (running as tests)..."

python/pylibcugraph/pylibcugraph/components/_connectivity.pxd

@@ -0,0 +1,40 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+# cython: language_level = 3
+
+from pylibcugraph.structure.graph_primtypes cimport *
+from pylibcugraph.structure.graph_utilities cimport *
+
+
+cdef extern from "cugraph/algorithms.hpp" namespace "cugraph":
+
+    ctypedef enum cugraph_cc_t:
+        CUGRAPH_WEAK "cugraph::cugraph_cc_t::CUGRAPH_WEAK"
+        CUGRAPH_STRONG "cugraph::cugraph_cc_t::CUGRAPH_STRONG"
+        NUM_CONNECTIVITY_TYPES "cugraph::cugraph_cc_t::NUM_CONNECTIVITY_TYPES"
+
+    cdef void connected_components[VT,ET,WT](
+        const GraphCSRView[VT,ET,WT] &graph,
+        cugraph_cc_t connect_type,
+        VT *labels) except +
+
+cdef extern from "cugraph/utilities/cython.hpp" namespace "cugraph::cython":
+    cdef void call_wcc[vertex_t, weight_t](
+        const handle_t &handle,
+        const graph_container_t &g,
+        vertex_t *identifiers) except +
+

python/pylibcugraph/pylibcugraph/components/_connectivity.pyx

@@ -11,10 +11,58 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from libc.stdint cimport uintptr_t
+from libcpp.memory cimport unique_ptr
 
-def strongly_connected_components(src, dst, weights, num_verts, num_edges, labels):
-    raise NotImplementedError()
+from pylibcugraph.components._connectivity cimport *
 
 
 def weakly_connected_components(src, dst, weights, num_verts, num_edges, labels):
-    raise NotImplementedError()
+
+    cdef unique_ptr[handle_t] handle_ptr
+    handle_ptr.reset(new handle_t())
+    handle_ = handle_ptr.get()
+
+    cdef uintptr_t c_src_vertices    = src['data'][0]
+    cdef uintptr_t c_dst_vertices    = dst['data'][0]
+    cdef uintptr_t c_edge_weights = <uintptr_t>NULL
+    cdef uintptr_t c_labels_val = labels['data'][0]
+
+    cdef graph_container_t graph_container
+    populate_graph_container(graph_container,
+                             handle_[0],
+                             <void*>c_src_vertices, <void*>c_dst_vertices, <void*>c_edge_weights,
+                             <void*>NULL,
+                             <void*>NULL,
+                             0,
+                             <numberTypeEnum>(<int>(numberTypeEnum.int32Type)),
+                             <numberTypeEnum>(<int>(numberTypeEnum.int32Type)),
+                             <numberTypeEnum>(<int>(numberTypeEnum.floatType)),
+                             num_edges,
+                             num_verts, num_edges,
+                             False,
+                             True,
+                             False,
+                             False)
+
+    call_wcc[int, float](handle_ptr.get()[0],
+                         graph_container,
+                         <int*> c_labels_val)
+
+
+def strongly_connected_components(offsets, indices, weights, num_verts, num_edges, labels):
+    cdef unique_ptr[handle_t] handle_ptr
+    handle_ptr.reset(new handle_t())
+    handle_ = handle_ptr.get()
+
+    cdef uintptr_t c_offsets    = offsets['data'][0]
+    cdef uintptr_t c_indices    = indices['data'][0]
+    cdef uintptr_t c_edge_weights = <uintptr_t>NULL
+    cdef uintptr_t c_labels_val = labels['data'][0]
+
+    cdef GraphCSRView[int,int,float] g
+
+    g = GraphCSRView[int,int,float](<int*>c_offsets, <int*>c_indices, <float*>NULL, num_verts, num_edges)
+
+    cdef cugraph_cc_t connect_type=CUGRAPH_STRONG
+    connected_components(g, <cugraph_cc_t>connect_type, <int *>c_labels_val)

python/pylibcugraph/pylibcugraph/structure/graph_primtypes.pxd

@@ -0,0 +1,81 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+# cython: language_level = 3
+
+from libcpp cimport bool
+from pylibcugraph.raft.common.handle cimport *
+
+cdef extern from "cugraph/legacy/graph.hpp" namespace "cugraph::legacy":
+
+    ctypedef enum PropType:
+        PROP_UNDEF "cugraph::legacy::PROP_UNDEF"
+        PROP_FALSE "cugraph::legacy::PROP_FALSE"
+        PROP_TRUE "cugraph::legacy::PROP_TRUE"
+
+    ctypedef enum DegreeDirection:
+        DIRECTION_IN_PLUS_OUT "cugraph::legacy::DegreeDirection::IN_PLUS_OUT"
+        DIRECTION_IN "cugraph::legacy::DegreeDirection::IN"
+        DIRECTION_OUT "cugraph::legacy::DegreeDirection::OUT"
+
+    struct GraphProperties:
+        bool directed
+        bool weighted
+        bool multigraph
+        bool bipartite
+        bool tree
+        PropType has_negative_edges
+
+    cdef cppclass GraphViewBase[VT,ET,WT]:
+        WT *edge_data
+        handle_t *handle;
+        GraphProperties prop
+        VT number_of_vertices
+        ET number_of_edges
+        VT* local_vertices
+        ET* local_edges
+        VT* local_offsets
+        void set_handle(handle_t*)
+        void set_local_data(VT* local_vertices_, ET* local_edges_, VT* local_offsets_)
+        void get_vertex_identifiers(VT *) const
+
+        GraphViewBase(WT*,VT,ET)
+
+    cdef cppclass GraphCOOView[VT,ET,WT](GraphViewBase[VT,ET,WT]):
+        VT *src_indices
+        VT *dst_indices
+
+        void degree(ET *,DegreeDirection) const
+
+        GraphCOOView()
+        GraphCOOView(const VT *, const ET *, const WT *, size_t, size_t)
+
+    cdef cppclass GraphCompressedSparseBaseView[VT,ET,WT](GraphViewBase[VT,ET,WT]):
+        ET *offsets
+        VT *indices
+
+        void get_source_indices(VT *) const
+        void degree(ET *,DegreeDirection) const
+
+        GraphCompressedSparseBaseView(const VT *, const ET *, const WT *, size_t, size_t)
+
+    cdef cppclass GraphCSRView[VT,ET,WT](GraphCompressedSparseBaseView[VT,ET,WT]):
+        GraphCSRView()
+        GraphCSRView(const VT *, const ET *, const WT *, size_t, size_t)
+
+    cdef cppclass GraphCSCView[VT,ET,WT](GraphCompressedSparseBaseView[VT,ET,WT]):
+        GraphCSCView()
+        GraphCSCView(const VT *, const ET *, const WT *, size_t, size_t)

python/pylibcugraph/pylibcugraph/structure/graph_utilities.pxd

@@ -0,0 +1,53 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+# cython: language_level = 3
+
+
+from pylibcugraph.raft.common.handle cimport *
+from libcpp cimport bool
+
+
+cdef extern from "cugraph/utilities/cython.hpp" namespace "cugraph::cython":
+
+    ctypedef enum numberTypeEnum:
+        int32Type "cugraph::cython::numberTypeEnum::int32Type"
+        int64Type "cugraph::cython::numberTypeEnum::int64Type"
+        floatType "cugraph::cython::numberTypeEnum::floatType"
+        doubleType "cugraph::cython::numberTypeEnum::doubleType"
+
+    cdef cppclass graph_container_t:
+       pass
+
+    cdef void populate_graph_container(
+        graph_container_t &graph_container,
+        handle_t &handle,
+        void *src_vertices,
+        void *dst_vertices,
+        void *weights,
+        void *vertex_partition_offsets,
+        void *segment_offsets,
+        size_t num_segments,
+        numberTypeEnum vertexType,
+        numberTypeEnum edgeType,
+        numberTypeEnum weightType,
+        size_t num_local_edges,
+        size_t num_global_vertices,
+        size_t num_global_edges,
+        bool is_weighted,
+        bool is_symmetric,
+        bool transposed,
+        bool multi_gpu) except +

python/pylibcugraph/pylibcugraph/tests/test_connected_components.py

@@ -12,15 +12,27 @@
 # limitations under the License.
 
 import pytest
+import numpy as np
+
+import cupy
+import cugraph
+
+from . import utils
 
 
 @pytest.fixture
-def plc():
+def package_under_test():
+    """
+    Create a fixture to import the package under test.  This is useful since
+    bugs that prevent the package under test from being imported will not
+    prevent pytest from collecting, listing, running, etc. the tests.
+    """
     import pylibcugraph
     return pylibcugraph
 
 
 ###############################################################################
+# Tests
 def test_import():
     """
     Ensure pylibcugraph is importable.
@@ -29,17 +41,58 @@ def test_import():
     import pylibcugraph  # noqa: F401
 
 
-def test_scc(plc):
+@pytest.mark.parametrize("graph_file", utils.DATASETS)
+def test_scc(package_under_test, graph_file):
     """
-    FIXME: rewrite once SCC is implemented.
+    Tests strongly_connected_components()
     """
-    with pytest.raises(NotImplementedError):
-        plc.strongly_connected_components(None, None, None, None, None, None)
+    pylibcugraph = package_under_test
+    cu_M = utils.read_csv_file(graph_file)
+    G = cugraph.DiGraph()
+    G.from_cudf_edgelist(cu_M, source="0", destination="1", edge_attr="2")
+
+    offsets, indices, weights = G.view_adj_list()
+    cupy_off = cupy.array(offsets)
+    cudf_ind = indices
 
+    cupy_labels = cupy.array(np.zeros(G.number_of_vertices()))
+    pylibcugraph.strongly_connected_components(
+        cupy_off.__cuda_array_interface__,
+        cudf_ind.__cuda_array_interface__,
+        None,
+        G.number_of_vertices(),
+        G.number_of_edges(directed_edges=True),
+        cupy_labels.__cuda_array_interface__
+    )
 
-def test_wcc(plc):
+    print(cupy_labels)
+    df = cugraph.strongly_connected_components(G)
+    print(df)
+
+
+@pytest.mark.parametrize("graph_file", utils.DATASETS)
+def test_wcc(package_under_test, graph_file):
     """
-    FIXME: rewrite once WCC is implemented.
+    Tests weakly_connected_components()
     """
-    with pytest.raises(NotImplementedError):
-        plc.weakly_connected_components(None, None, None, None, None, None)
+    pylibcugraph = package_under_test
+    cu_M = utils.read_csv_file(graph_file)
+    G = cugraph.DiGraph()
+    G.from_cudf_edgelist(cu_M, source="0", destination="1", edge_attr="2")
+
+    cupy_src = cupy.array(cu_M["0"])
+    cudf_dst = cu_M["1"]
+
+    cupy_labels = cupy.array(np.zeros(G.number_of_vertices()), dtype='int32')
+    pylibcugraph.weakly_connected_components(
+        cupy_src.__cuda_array_interface__,
+        cudf_dst.__cuda_array_interface__,
+        None,
+        G.number_of_vertices(),
+        G.number_of_edges(directed_edges=True),
+        cupy_labels.__cuda_array_interface__
+    )
+
+    print(cupy_labels)
+    df = cugraph.weakly_connected_components(G)
+    print(df)