From 55ecf5c3d17eb0fa859daa795f56d82e5a978d46 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Mon, 17 Jan 2022 00:46:44 -0600 Subject: [PATCH 01/43] Start of GPUGraphData class and test, added 'experimental' decorator. --- python/pylibcugraph/pylibcugraph/__init__.py | 5 + .../pylibcugraph/structure/gpu_graph_data.py | 33 +++ .../pylibcugraph/tests/test_gpu_graph_data.py | 36 ++++ .../pylibcugraph/utilities/__init__.py | 1 + .../pylibcugraph/utilities/experimental.py | 195 ++++++++++++++++++ 5 files changed, 270 insertions(+) create mode 100644 python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py create mode 100644 python/pylibcugraph/pylibcugraph/tests/test_gpu_graph_data.py create mode 100644 python/pylibcugraph/pylibcugraph/utilities/__init__.py create mode 100644 python/pylibcugraph/pylibcugraph/utilities/experimental.py diff --git a/python/pylibcugraph/pylibcugraph/__init__.py b/python/pylibcugraph/pylibcugraph/__init__.py index 868d5d779da..25d10665fe5 100644 --- a/python/pylibcugraph/pylibcugraph/__init__.py +++ b/python/pylibcugraph/pylibcugraph/__init__.py @@ -15,3 +15,8 @@ strongly_connected_components, weakly_connected_components, ) + +# Import gpu_graph_data to add objects to experimental. +# Once moved out of experimental, import individual objects here to include in +# this namespace. +import pylibcugraph.structure.gpu_graph_data diff --git a/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py b/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py new file mode 100644 index 00000000000..f1466f27e6b --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py @@ -0,0 +1,33 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph.utilities.experimental import experimental + +@experimental(ns_name="") +class __GPUGraphData: + """ + Data that resides on one or more GPUs that represents a graph. + + GPUGraphData instances define the mapping between user-provided data that + represents a graph (vertex IDs, edge lists) and the corresponding + representation needed to support GPU-based graph operations (0-based vertex + array indices, CSR, CSC, etc.). + + Deleting a GPUGraphData intance frees the GPU memory used for storing the + graph, but does not delete the src, dst, and weights arrays used to + construct the instance. + """ + + def __init__(self, src_array, dst_array, weight_array, + store_transposed=False): + self.__graph = None diff --git a/python/pylibcugraph/pylibcugraph/tests/test_gpu_graph_data.py b/python/pylibcugraph/pylibcugraph/tests/test_gpu_graph_data.py new file mode 100644 index 00000000000..2a536403b80 --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/tests/test_gpu_graph_data.py @@ -0,0 +1,36 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pandas as pd +import cupy as cp +import numpy as np + +from . import utils + + +def test_ctor(): + from pylibcugraph.experimental import GPUGraphData + + pdf = pd.read_csv(utils.RAPIDS_DATASET_ROOT_DIR_PATH/"karate.csv", + delimiter=" ", header=None, + names=["0", "1", "weight"], + dtype={"0": "int32", "1": "int32", "weight": "float32"}, + ) + device_srcs = cp.asarray(pdf["0"].to_numpy(), dtype=np.int32) + device_dsts = cp.asarray(pdf["1"].to_numpy(), dtype=np.int32) + device_weights = cp.asarray(pdf["weight"].to_numpy(), dtype=np.float32) + + G = GPUGraphData(src_array=device_srcs, + dst_array=device_dsts, + weight_array=device_weights, + store_transposed=False) diff --git a/python/pylibcugraph/pylibcugraph/utilities/__init__.py b/python/pylibcugraph/pylibcugraph/utilities/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/utilities/__init__.py @@ -0,0 +1 @@ + diff --git a/python/pylibcugraph/pylibcugraph/utilities/experimental.py b/python/pylibcugraph/pylibcugraph/utilities/experimental.py new file mode 100644 index 00000000000..439abc7b3aa --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/utilities/experimental.py @@ -0,0 +1,195 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import types +import warnings +import sys + +import pylibcugraph + +def add_obj_to_pylibcugraph_subnamespace(obj, sub_ns_name, + sub_sub_ns_name=None, + new_obj_name=None): + """ + Adds an obj to the pylibcugraph. namespace, using the + objects current namespace names under pylibcugraph as the default + sub_sub_ns_name. + + Example: + for pylibcugraph.link_analysis.pagerank.pagerank: + add_obj_to_pylibcugraph_subnamespace(pagerank, "experimental") + results in: + pylibcugraph.experimental.link_analysis.pagerank.pagerank + + All namespaces - current and new - must be under "pylibcugraph". + + If sub_sub_ns_name is provided, it will be used to override the obj's + current namespace under pylibcugraph. + Example: + for pylibcugraph.link_analysis.pagerank.pagerank: + add_obj_to_pylibcugraph_subnamespace(pagerank, "experimental", + sub_sub_ns_name="foo.bar") + results in: + pylibcugraph.experimental.foo.bar.pagerank + + If new_obj_name is provided, it will be used to renamed the obj in the new + namespace. + Example: + for pylibcugraph.link_analysis.pagerank.pagerank: + add_obj_to_pylibcugraph_subnamespace(pagerank, "experimental", + new_obj_name="new_pagerank") + results in: + pylibcugraph.experimental.link_analysis.pagerank.new_pagerank + + Returns a tuple of: + (new namespace name, new obj name, new namespace module obj) + """ + # Create a list of names representing the ns to create + current_mod_name_parts = obj.__module__.split(".") + # All namespaces - current and new - must be under "pylibcugraph" + if current_mod_name_parts[0] != "pylibcugraph": + raise ValueError(f"{obj.__name__} is not under the pylibcugraph " + "package") + new_mod_name_parts = [sub_ns_name] + if sub_sub_ns_name is None: + new_mod_name_parts += current_mod_name_parts[1:] + else: + if sub_sub_ns_name != "": + new_mod_name_parts += sub_sub_ns_name.split(".") + + # Create the new namespace + mod_to_update = pylibcugraph + mod_name_parts = ["pylibcugraph"] + + for ns in new_mod_name_parts: + mod_name_parts.append(ns) + if not(hasattr(mod_to_update, ns)): + mod_to_update_name = ".".join(mod_name_parts) + new_mod = types.ModuleType(mod_to_update_name) + setattr(mod_to_update, ns, new_mod) + sys.modules[mod_to_update_name] = new_mod + mod_to_update = getattr(mod_to_update, ns) + + # Add obj to the new namespace + new_obj_name = obj.__name__ if new_obj_name is None else new_obj_name + setattr(mod_to_update, new_obj_name, obj) + + return (".".join(["pylibcugraph"] + new_mod_name_parts), + new_obj_name, + mod_to_update) + + +def experimental(*args, **kwargs): + """ + Decorator function to add an obj to the pylibcugraph.experimental + namespace. + + If no args are given, obj is copied to + pylibcugraph.experimental..obj. + + Example: + for the pagerank function in pylibcugraph.link_analysis.pagerank: + @experimental + def pagerank(...) + results in: + a pagerank() function in the + pylibcugraph.experimental.link_analysis.pagerank namespace. + + If the ns_name kwarg is given, it is used to replace the default + subnamespace under pylibcugraph that the obj currently resides in. + Example: + for the pagerank function in pylibcugraph.link_analysis.pagerank: + @experimental(ns_name="foo") + def pagerank(...) + results in: + a pagerank() function in the + pylibcugraph.experimental.foo namespace. + + Setting ns_name="" results in the obj being added directly to the + pylibcugraph.experimental namespace. + + If the current obj is private by naming it with a leading __, the leading + __ is removed from the obj in the new namespace. This allows an + experimental class/function to be private (hidden) in a non-experimental + namespace but public in experimental. + Example: + for the __pagerank function in pylibcugraph.link_analysis.pagerank: + @experimental(ns_name="foo") + def __pagerank(...) + results in: + a pagerank() function in the + pylibcugraph.experimental.foo namespace. + """ + kwa = list(kwargs.keys()) + if kwa and kwa != ["ns_name"]: + raise TypeError("Only the 'ns_name' kwarg is allowed for " + "experimental()") + ns_name = kwargs.get("ns_name") + + # python expects decorators to return function wrappers one of two ways: if + # args specified, the decorator must return a callable that accepts an obj + # to wrap. If no args, then the decorator returns the wrapped obj directly. + if ns_name is not None: # called as @experimental(ns_name="...") + sub_namespace_name = ns_name + # The function to return that takes the obj and sets up the new ns + def experimental_ns_updater(obj): + (new_ns_name, new_obj_name, new_ns) = \ + add_obj_to_pylibcugraph_subnamespace( + obj, "experimental", + sub_sub_ns_name=sub_namespace_name, + new_obj_name=obj.__name__.lstrip("__")) + # Wrap the function in a function that prints a warning before + # calling the obj. This is done after adding obj to the + # experimental namespace so the warning message would have the + # properly-generated experimental names. + warning_msg = (f"{new_ns_name}.{new_obj_name} is experimental and " + "will change in a future release.") + obj.__module__ = new_ns_name + obj.__qualname__ = new_obj_name + @functools.wraps(obj) + def call_with_warning(*args, **kwargs): + warnings.warn(warning_msg, PendingDeprecationWarning) + return obj(*args, **kwargs) + + wrapped_obj = call_with_warning + # Replace obj in the experimental ns with the wrapped obj + setattr(new_ns, new_obj_name, wrapped_obj) + return obj + return experimental_ns_updater + + else: # called as @experimental + if len(args) > 1: + raise TypeError("Too many positional args to experimental()") + obj = args[0] + (new_ns_name, new_obj_name, new_ns) = \ + add_obj_to_pylibcugraph_subnamespace( + obj, "experimental", + new_obj_name=obj.__name__.lstrip("__")) + # Wrap the function in a function that prints a warning before calling + # the obj. This is done after adding obj to the experimental namespace + # so the warning message would have the properly-generated experimental + # names. + warning_msg = (f"{new_ns_name}.{new_obj_name} is experimental and " + "will change in a future release.") + obj.__module__ = new_ns_name + obj.__qualname__ = new_obj_name + @functools.wraps(obj) + def call_with_warning(*args, **kwargs): + warnings.warn(warning_msg, PendingDeprecationWarning) + return obj(*args, **kwargs) + + wrapped_obj = call_with_warning + # Replace obj in the experimental ns with the wrapped obj + setattr(new_ns, new_obj_name, wrapped_obj) + return obj From 9814793ca9ff2608c9b7fde3b369e84838cdf394 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Mon, 17 Jan 2022 14:33:21 -0600 Subject: [PATCH 02/43] Initial pylibcugraph cython files for cugraph_c API, still WIP. --- .../pylibcugraph/_cugraph_c/__init__.py | 0 .../pylibcugraph/_cugraph_c/_algorithms.pxd | 81 ++++++++++++++++ .../pylibcugraph/_cugraph_c/_array.pxd | 95 +++++++++++++++++++ .../pylibcugraph/_cugraph_c/_cugraph_api.pxd | 41 ++++++++ .../pylibcugraph/_cugraph_c/_error.pxd | 37 ++++++++ .../pylibcugraph/_cugraph_c/_graph.pxd | 77 +++++++++++++++ .../pylibcugraph/_cugraph_c/_graph.pyx | 62 ++++++++++++ .../pylibcugraph/structure/gpu_graph_data.py | 3 + .../pylibcugraph/tests/test_gpu_graph_data.py | 29 +++++- .../pylibcugraph/pylibcugraph/tests/utils.py | 62 +++++++++++- 10 files changed, 479 insertions(+), 8 deletions(-) create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/__init__.py create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pyx diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/__init__.py b/python/pylibcugraph/pylibcugraph/_cugraph_c/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd new file mode 100644 index 00000000000..a432680d77b --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd @@ -0,0 +1,81 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph._cugraph_c._cugraph_api cimport ( + bool_t, + cugraph_resource_handle_t, +) +from pylibcugraph._cugraph_c._error cimport ( + cugraph_error_code_t, + cugraph_error_t, +) +from pylibcugraph._cugraph_c._array cimport ( + cugraph_type_erased_device_array_t, +) + + +cdef extern from "cugraph_c/algorithms.h": + + struct cugraph_pagerank_result_t: + int _placeholder + + # struct cugraph_paths_result_t: + # int _placeholder + + # struct cugraph_extract_paths_result_t: + # int _placeholder + + cdef cugraph_type_erased_device_array_t* + cugraph_pagerank_result_get_vertices( + cugraph_pagerank_result_t* result + ) + + cdef cugraph_type_erased_device_array_t* + cugraph_pagerank_result_get_pageranks( + cugraph_pagerank_result_t* result + ) + + cdef void + cugraph_pagerank_result_free( + cugraph_pagerank_result_t* result + ) + + cdef cugraph_error_code_t + cugraph_pagerank( + const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + const cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, + double alpha, + double epsilon, + size_t max_iterations, + bool_t has_initial_guess, + bool_t do_expensive_check, + cugraph_pagerank_result_t** result, + cugraph_error_t** error + ) + + cdef cugraph_error_code_t + cugraph_personalized_pagerank( + const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + const cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, + cugraph_type_erased_device_array_t* personalization_vertices, + const cugraph_type_erased_device_array_t* personalization_values, + double alpha, + double epsilon, + size_t max_iterations, + bool_t has_initial_guess, + bool_t do_expensive_check, + cugraph_pagerank_result_t** result, + cugraph_error_t** error + ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd new file mode 100644 index 00000000000..355e8f3eadf --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd @@ -0,0 +1,95 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph._cugraph_c._error cimport ( + cugraph_error_code_t, + cugraph_error_t, +) +from pylibcugraph._cugraph_c._cugraph_api cimport ( + cugraph_resource_handle_t, + data_type_id_t, + byte_t, +) + + +cdef extern from "cugraph_c/array.h": + + struct cugraph_type_erased_device_array_t: + int _placeholder + + struct cugraph_type_erased_host_array_t: + int _placeholder + + cdef cugraph_error_code_t \ + cugraph_type_erased_device_array_create( + const cugraph_resource_handle_t* handle, + data_type_id_t dtype, + size_t n_elems, + cugraph_type_erased_device_array_t** array, + cugraph_error_t** error + ) + + cdef void \ + cugraph_type_erased_device_array_free( + cugraph_type_erased_device_array_t* p + ) + + cdef size_t \ + cugraph_type_erased_device_array_size( + const cugraph_type_erased_device_array_t* p + ) + + cdef data_type_id_t \ + cugraph_type_erased_device_array_type( + const cugraph_type_erased_device_array_t* p + ) + + cdef cugraph_error_code_t \ + cugraph_type_erased_host_array_create( + const cugraph_resource_handle_t* handle, + data_type_id_t dtype, + size_t n_elems, + cugraph_type_erased_host_array_t** array, + cugraph_error_t** error + ) + + cdef void \ + cugraph_type_erased_host_array_free( + cugraph_type_erased_host_array_t* p + ) + + cdef size_t \ + cugraph_type_erased_host_array_size( + const cugraph_type_erased_host_array_t* p + ) + + cdef data_type_id_t \ + cugraph_type_erased_host_array_type( + const cugraph_type_erased_host_array_t* p + ) + + cdef cugraph_error_code_t \ + cugraph_type_erased_device_array_copy_from_host( + const cugraph_resource_handle_t* handle, + cugraph_type_erased_device_array_t* dst, + const byte_t* h_src, + cugraph_error_t** error + ) + + cdef cugraph_error_code_t \ + cugraph_type_erased_device_array_copy_to_host( + const cugraph_resource_handle_t* handle, + byte_t* h_dst, + const cugraph_type_erased_device_array_t* src, + cugraph_error_t** error + ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd new file mode 100644 index 00000000000..e8dc5adcc92 --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd @@ -0,0 +1,41 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libc.stdint cimport int8_t + + +cdef extern from "cugraph_c/cugraph_api.h": + + ctypedef enum bool_t: + FALSE + TRUE + + ctypedef enum data_type_id_t: + INT32 + INT64 + FLOAT32 + FLOAT64 + NTYPES + + ctypedef int8_t byte_t + + struct cugraph_resource_handle_t: + int _placeholder + + cdef cugraph_resource_handle_t* \ + cugraph_create_resource_handle() + + cdef void \ + cugraph_free_resource_handle( + cugraph_resource_handle_t* p_handle + ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd new file mode 100644 index 00000000000..60ee464225e --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd @@ -0,0 +1,37 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +cdef extern from "cugraph_c/error.h": + + ctypedef enum cugraph_error_code_t: + CUGRAPH_SUCCESS + CUGRAPH_UNKNOWN_ERROR + CUGRAPH_INVALID_HANDLE + CUGRAPH_ALLOC_ERROR + CUGRAPH_INVALID_INPUT + CUGRAPH_NOT_IMPLEMENTED + CUGRAPH_UNSUPPORTED_TYPE_COMBINATION + + struct cugraph_error_t: + int _placeholder + + const char* \ + cugraph_error_message( + const cugraph_error_t* error + ) + + void \ + cugraph_error_free( + cugraph_error_t* error + ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd new file mode 100644 index 00000000000..d6280259e5c --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd @@ -0,0 +1,77 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph._cugraph_c._cugraph_api cimport ( + bool_t, + cugraph_resource_handle_t, +) +from pylibcugraph._cugraph_c._error cimport ( + cugraph_error_code_t, + cugraph_error_t, +) +from pylibcugraph._cugraph_c._array cimport ( + cugraph_type_erased_device_array_t, + cugraph_type_erased_host_array_t, +) + + +cdef extern from "cugraph_c/graph.h": + + struct cugraph_graph_t: + int _placeholder + + struct cugraph_graph_properties_t: + bool_t is_symmetric + bool_t is_multigraph + + cdef cugraph_error_code_t \ + cugraph_sg_graph_create( + const cugraph_resource_handle_t* handle, + const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_t* src, + const cugraph_type_erased_device_array_t* dst, + const cugraph_type_erased_device_array_t* weights, + bool_t store_transposed, + bool_t renumber, + bool_t check, + cugraph_graph_t** graph, + cugraph_error_t** error) + + # This may get renamed to cugraph_graph_free() + cdef void \ + cugraph_sg_graph_free( + cugraph_graph_t* graph + ) + + cdef cugraph_error_code_t \ + cugraph_mg_graph_create( + const cugraph_resource_handle_t* handle, + const cugraph_graph_properties_t* properties, + const cugraph_type_erased_device_array_t* src, + const cugraph_type_erased_device_array_t* dst, + const cugraph_type_erased_device_array_t* weights, + const cugraph_type_erased_host_array_t* vertex_partition_offsets, + const cugraph_type_erased_host_array_t* segment_offsets, + bool_t store_transposed, + size_t num_vertices, + size_t num_edges, + bool_t check, + cugraph_graph_t** graph, + cugraph_error_t** error + ) + + # This may get renamed to or replaced with cugraph_graph_free() + cdef void \ + cugraph_mg_graph_free( + cugraph_graph_t* graph + ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pyx new file mode 100644 index 00000000000..2266e783f14 --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pyx @@ -0,0 +1,62 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libc.stdint cimport uintptr_t + +from pylibcugraph._cugraph_c._cugraph_api cimport ( + bool_t, + cugraph_resource_handle_t, +) +from pylibcugraph._cugraph_c._error cimport ( + cugraph_error_code_t, + cugraph_error_t, +) +from pylibcugraph._cugraph_c._array cimport ( + cugraph_type_erased_device_array_t, +) +from pylibcugraph._cugraph_c._graph cimport ( + cugraph_sg_graph_create as _cugraph_sg_graph_create, + cugraph_graph_properties_t +) + + +def cugraph_sg_graph_create( + resource_handle, + graph_properties, + src_array, + dst_array, + weight_array, + store_transposed, + renumber, + expensive_check): + # use __cuda_array_interface__ to get device pointers + + cdef uintptr_t x = 0; + cdef cugraph_resource_handle_t* rh + cdef cugraph_graph_properties_t* p + cdef cugraph_type_erased_device_array_t* a + cdef cugraph_graph_t* g + cdef cugraph_error_t* e + cdef bool_t true=bool_t.TRUE + cdef bool_t false=bool_t.FALSE + + _cugraph_sg_graph_create(rh, + p, + a, + a, + a, + true, + false, + false, + &g, + &e) diff --git a/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py b/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py index f1466f27e6b..90e5756582a 100644 --- a/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py +++ b/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py @@ -12,6 +12,7 @@ # limitations under the License. from pylibcugraph.utilities.experimental import experimental +from pylibcugraph._cugraph_c._graph import cugraph_sg_graph_create @experimental(ns_name="") class __GPUGraphData: @@ -31,3 +32,5 @@ class __GPUGraphData: def __init__(self, src_array, dst_array, weight_array, store_transposed=False): self.__graph = None + + cugraph_sg_graph_create(None,None,None,None,None,None,None,None) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_gpu_graph_data.py b/python/pylibcugraph/pylibcugraph/tests/test_gpu_graph_data.py index 2a536403b80..1f96d16fa36 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_gpu_graph_data.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_gpu_graph_data.py @@ -15,13 +15,24 @@ import cupy as cp import numpy as np +import pytest + from . import utils -def test_ctor(): - from pylibcugraph.experimental import GPUGraphData +# ============================================================================= +# Pytest fixtures +# ============================================================================= +datasets = [utils.RAPIDS_DATASET_ROOT_DIR_PATH/"karate.csv", + utils.RAPIDS_DATASET_ROOT_DIR_PATH/"dolphins.csv", + ] + +@pytest.fixture(scope="module", + params=[pytest.param(ds, id=ds.name) for ds in datasets]) +def graph_arrays(request): + ds = request.param - pdf = pd.read_csv(utils.RAPIDS_DATASET_ROOT_DIR_PATH/"karate.csv", + pdf = pd.read_csv(ds, delimiter=" ", header=None, names=["0", "1", "weight"], dtype={"0": "int32", "1": "int32", "weight": "float32"}, @@ -30,7 +41,19 @@ def test_ctor(): device_dsts = cp.asarray(pdf["1"].to_numpy(), dtype=np.int32) device_weights = cp.asarray(pdf["weight"].to_numpy(), dtype=np.float32) + return (device_srcs, device_dsts, device_weights) + + +############################################################################### +# Tests +def test_ctor(graph_arrays): + from pylibcugraph.experimental import GPUGraphData + + (device_srcs, device_dsts, device_weights) = graph_arrays + G = GPUGraphData(src_array=device_srcs, dst_array=device_dsts, weight_array=device_weights, store_transposed=False) + + # FIXME: test for correct num verts, edges, etc. diff --git a/python/pylibcugraph/pylibcugraph/tests/utils.py b/python/pylibcugraph/pylibcugraph/tests/utils.py index 2856fb8af5a..03ddcd3c534 100644 --- a/python/pylibcugraph/pylibcugraph/tests/utils.py +++ b/python/pylibcugraph/pylibcugraph/tests/utils.py @@ -13,7 +13,9 @@ import os from pathlib import Path +from itertools import product +import pytest import cudf @@ -21,11 +23,6 @@ RAPIDS_DATASET_ROOT_DIR_PATH = Path(RAPIDS_DATASET_ROOT_DIR) -DATASETS = [RAPIDS_DATASET_ROOT_DIR_PATH/f for f in [ - "karate.csv", - "dolphins.csv"]] - - def read_csv_file(csv_file, weights_dtype="float32"): return cudf.read_csv( csv_file, @@ -33,3 +30,58 @@ def read_csv_file(csv_file, weights_dtype="float32"): dtype=["int32", "int32", weights_dtype], header=None, ) + + +def genFixtureParamsProduct(*args): + """ + Returns the cartesian product of the param lists passed in. The lists must + be flat lists of pytest.param objects, and the result will be a flat list + of pytest.param objects with values and meta-data combined accordingly. A + flat list of pytest.param objects is required for pytest fixtures to + properly recognize the params. The combinations also include ids generated + from the param values and id names associated with each list. For example: + + genFixtureParamsProduct( ([pytest.param(True, marks=[pytest.mark.A_good]), + pytest.param(False, marks=[pytest.mark.A_bad])], + "A"), + ([pytest.param(True, marks=[pytest.mark.B_good]), + pytest.param(False, marks=[pytest.mark.B_bad])], + "B") ) + + results in fixture param combinations: + + True, True - marks=[A_good, B_good] - id="A=True,B=True" + True, False - marks=[A_good, B_bad] - id="A=True,B=False" + False, True - marks=[A_bad, B_good] - id="A=False,B=True" + False, False - marks=[A_bad, B_bad] - id="A=False,B=False" + + Simply using itertools.product on the lists would result in a list of + sublists of individual param objects (ie. not "merged"), which would not be + recognized properly as params for a fixture by pytest. + + NOTE: This function is only needed for parameterized fixtures. + Tests/benchmarks will automatically get this behavior when specifying + multiple @pytest.mark.parameterize(param_name, param_value_list) + decorators. + """ + # Ensure each arg is a list of pytest.param objs, then separate the params + # and IDs. + paramLists = [] + ids = [] + paramType = pytest.param().__class__ + for (paramList, id) in args: + for i in range(len(paramList)): + if not isinstance(paramList[i], paramType): + paramList[i] = pytest.param(paramList[i]) + paramLists.append(paramList) + ids.append(id) + + retList = [] + for paramCombo in product(*paramLists): + values = [p.values[0] for p in paramCombo] + marks = [m for p in paramCombo for m in p.marks] + comboid = ",".join( + ["%s=%s" % (id, p.values[0]) for (p, id) in zip(paramCombo, ids)] + ) + retList.append(pytest.param(values, marks=marks, id=comboid)) + return retList From ca784da1ea0cf07ee6c1cea885b09d3279406460 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Mon, 17 Jan 2022 14:59:54 -0600 Subject: [PATCH 03/43] pylibcugraph C API cython code cythonizes and compiles, new bindings can be imported. --- .../pylibcugraph/_cugraph_c/_algorithms.pxd | 12 ++++++------ .../pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd | 8 ++++---- .../pylibcugraph/_cugraph_c/_cugraph_api.pxd | 4 ++-- .../pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd | 4 ++-- .../pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd | 6 +++--- python/pylibcugraph/setup.py | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd index a432680d77b..0a1a5ffda45 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd @@ -26,14 +26,14 @@ from pylibcugraph._cugraph_c._array cimport ( cdef extern from "cugraph_c/algorithms.h": - struct cugraph_pagerank_result_t: - int _placeholder + ctypedef struct cugraph_pagerank_result_t: + pass - # struct cugraph_paths_result_t: - # int _placeholder + # ctypedef struct cugraph_paths_result_t: + # pass - # struct cugraph_extract_paths_result_t: - # int _placeholder + # ctypedef struct cugraph_extract_paths_result_t: + # pass cdef cugraph_type_erased_device_array_t* cugraph_pagerank_result_get_vertices( diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd index 355e8f3eadf..0366a94655e 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd @@ -24,11 +24,11 @@ from pylibcugraph._cugraph_c._cugraph_api cimport ( cdef extern from "cugraph_c/array.h": - struct cugraph_type_erased_device_array_t: - int _placeholder + ctypedef struct cugraph_type_erased_device_array_t: + pass - struct cugraph_type_erased_host_array_t: - int _placeholder + ctypedef struct cugraph_type_erased_host_array_t: + pass cdef cugraph_error_code_t \ cugraph_type_erased_device_array_create( diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd index e8dc5adcc92..7b4299e550d 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd @@ -29,8 +29,8 @@ cdef extern from "cugraph_c/cugraph_api.h": ctypedef int8_t byte_t - struct cugraph_resource_handle_t: - int _placeholder + ctypedef struct cugraph_resource_handle_t: + pass cdef cugraph_resource_handle_t* \ cugraph_create_resource_handle() diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd index 60ee464225e..1610deb41cd 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd @@ -23,8 +23,8 @@ cdef extern from "cugraph_c/error.h": CUGRAPH_NOT_IMPLEMENTED CUGRAPH_UNSUPPORTED_TYPE_COMBINATION - struct cugraph_error_t: - int _placeholder + ctypedef struct cugraph_error_t: + pass const char* \ cugraph_error_message( diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd index d6280259e5c..3dc3bfdc0d9 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd @@ -27,10 +27,10 @@ from pylibcugraph._cugraph_c._array cimport ( cdef extern from "cugraph_c/graph.h": - struct cugraph_graph_t: - int _placeholder + ctypedef struct cugraph_graph_t: + pass - struct cugraph_graph_properties_t: + ctypedef struct cugraph_graph_properties_t: bool_t is_symmetric bool_t is_multigraph diff --git a/python/pylibcugraph/setup.py b/python/pylibcugraph/setup.py index 74835d2ac32..d4ce8fb6793 100644 --- a/python/pylibcugraph/setup.py +++ b/python/pylibcugraph/setup.py @@ -130,7 +130,7 @@ def run(self): cuda_lib_dir, os.path.join(os.sys.prefix, "lib") ], - libraries=['cudart', 'cusparse', 'cusolver', 'cugraph', 'nccl'], + libraries=['cudart', 'cusparse', 'cusolver', 'cugraph', 'nccl', 'cugraph_c'], language='c++', extra_compile_args=['-std=c++17']) ] From 56e189f7bfea80c47653c46c24badbe8d520b339 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 18 Jan 2022 15:45:09 -0600 Subject: [PATCH 04/43] Added initial RAII-style cdef classes (not building yet). --- .../pylibcugraph/_cugraph_c/_graph.pyx | 62 ---------------- .../_cugraph_c/_resource_handle.pyx | 34 +++++++++ .../pylibcugraph/_cugraph_c/_sg_graph.pyx | 71 +++++++++++++++++++ .../pylibcugraph/structure/gpu_graph_data.py | 12 +++- 4 files changed, 115 insertions(+), 64 deletions(-) delete mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pyx create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/_resource_handle.pyx create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/_sg_graph.pyx diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pyx deleted file mode 100644 index 2266e783f14..00000000000 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pyx +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from libc.stdint cimport uintptr_t - -from pylibcugraph._cugraph_c._cugraph_api cimport ( - bool_t, - cugraph_resource_handle_t, -) -from pylibcugraph._cugraph_c._error cimport ( - cugraph_error_code_t, - cugraph_error_t, -) -from pylibcugraph._cugraph_c._array cimport ( - cugraph_type_erased_device_array_t, -) -from pylibcugraph._cugraph_c._graph cimport ( - cugraph_sg_graph_create as _cugraph_sg_graph_create, - cugraph_graph_properties_t -) - - -def cugraph_sg_graph_create( - resource_handle, - graph_properties, - src_array, - dst_array, - weight_array, - store_transposed, - renumber, - expensive_check): - # use __cuda_array_interface__ to get device pointers - - cdef uintptr_t x = 0; - cdef cugraph_resource_handle_t* rh - cdef cugraph_graph_properties_t* p - cdef cugraph_type_erased_device_array_t* a - cdef cugraph_graph_t* g - cdef cugraph_error_t* e - cdef bool_t true=bool_t.TRUE - cdef bool_t false=bool_t.FALSE - - _cugraph_sg_graph_create(rh, - p, - a, - a, - a, - true, - false, - false, - &g, - &e) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_resource_handle.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/_resource_handle.pyx new file mode 100644 index 00000000000..052dd40e68b --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_resource_handle.pyx @@ -0,0 +1,34 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph._cugraph_c._cugraph_api cimport ( + cugraph_resource_handle_t, + cugraph_create_resource_handle, + cugraph_free_resource_handle, +) + + +cdef class ResourceHandle: + """ + RAII-stye resource handle class to manage individual create/free calls and + the corresponding pointer to a cugraph_resource_handle_t + """ + cdef cugraph_resource_handle_t* __handle + + def __cinit__(self): + self.__handle = cugraph_create_resource_handle() + # FIXME: check for error + + def __dealloc__(self): + # FIXME: free only if handle is a valid pointer + cugraph_free_resource_handle(self.__handle) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_sg_graph.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/_sg_graph.pyx new file mode 100644 index 00000000000..1f4fcda268f --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_sg_graph.pyx @@ -0,0 +1,71 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libc.stdint cimport uintptr_t + +from pylibcugraph._cugraph_c._cugraph_api cimport ( + bool_t, + cugraph_resource_handle_t, +) +from pylibcugraph._cugraph_c._error cimport ( + cugraph_error_code_t, + cugraph_error_t, +) +from pylibcugraph._cugraph_c._array cimport ( + cugraph_type_erased_device_array_t, +) +from pylibcugraph._cugraph_c._graph cimport ( + cugraph_sg_graph_create, + cugraph_graph_properties_t, + cugraph_sg_graph_free, +) + + +cdef class SGGraph: + """ + RAII-stye Graph class for use with single-GPU APIs that manages the + individual create/free calls and the corresponding cugraph_graph_t pointer. + """ + def __cinit__(self, + resource_handle, + graph_properties, + src_array, + dst_array, + weight_array, + store_transposed, + renumber, + expensive_check): + # use __cuda_array_interface__ to get device pointers + + cdef uintptr_t x = 0; + cdef cugraph_resource_handle_t* rh + cdef cugraph_graph_properties_t* p + cdef cugraph_type_erased_device_array_t* a + cdef cugraph_graph_t* self.__sg_graph + cdef cugraph_error_t* e + cdef bool_t true=bool_t.TRUE + cdef bool_t false=bool_t.FALSE + + cugraph_sg_graph_create(rh, + p, + a, + a, + a, + true, + false, + false, + &g, + &e) + + def __dealloc__(self): + cugraph_sg_graph_free(self.__sg_graph) diff --git a/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py b/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py index 90e5756582a..086c83f14f8 100644 --- a/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py +++ b/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py @@ -13,6 +13,8 @@ from pylibcugraph.utilities.experimental import experimental from pylibcugraph._cugraph_c._graph import cugraph_sg_graph_create +from pylibcugraph._cugraph_c._resource_handle import ResourceHandle + @experimental(ns_name="") class __GPUGraphData: @@ -30,7 +32,13 @@ class __GPUGraphData: """ def __init__(self, src_array, dst_array, weight_array, - store_transposed=False): + store_transposed=False, + resource_handle=None): self.__graph = None + if resource_handle is None: + self.__resource_handle = ResourceHandle() + else: + self.__resource_handle = resource_handle - cugraph_sg_graph_create(None,None,None,None,None,None,None,None) + cugraph_sg_graph_create(self.__resource_handle, + None,None,None,None,None,None,None) From f3d2c4966f470899565c83fa775b7ea44d1d886a Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 18 Jan 2022 16:41:30 -0600 Subject: [PATCH 05/43] Renamed files under _cugraph_c to drop the leading underscore since it probably is not necessary given the dir name, updated @experimental decorator to not error on extension types (from cython), removed unnecessary gpu_graph_data class and replaced with cython SGGraph class. --- python/pylibcugraph/pylibcugraph/__init__.py | 13 ++++-- .../{_algorithms.pxd => algorithms.pxd} | 6 +-- .../_cugraph_c/{_array.pxd => array.pxd} | 4 +- .../{_cugraph_api.pxd => cugraph_api.pxd} | 0 .../_cugraph_c/{_error.pxd => error.pxd} | 0 .../_cugraph_c/{_graph.pxd => graph.pxd} | 6 +-- ...esource_handle.pyx => resource_handle.pyx} | 2 +- .../{_sg_graph.pyx => sg_graph.pyx} | 16 ++++--- .../pylibcugraph/structure/gpu_graph_data.py | 44 ------------------- ...est_gpu_graph_data.py => test_sg_graph.py} | 10 ++--- .../pylibcugraph/utilities/experimental.py | 20 ++++++--- 11 files changed, 47 insertions(+), 74 deletions(-) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{_algorithms.pxd => algorithms.pxd} (94%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{_array.pxd => array.pxd} (96%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{_cugraph_api.pxd => cugraph_api.pxd} (100%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{_error.pxd => error.pxd} (100%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{_graph.pxd => graph.pxd} (94%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{_resource_handle.pyx => resource_handle.pyx} (95%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{_sg_graph.pyx => sg_graph.pyx} (84%) delete mode 100644 python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py rename python/pylibcugraph/pylibcugraph/tests/{test_gpu_graph_data.py => test_sg_graph.py} (88%) diff --git a/python/pylibcugraph/pylibcugraph/__init__.py b/python/pylibcugraph/pylibcugraph/__init__.py index 25d10665fe5..5383a66e48e 100644 --- a/python/pylibcugraph/pylibcugraph/__init__.py +++ b/python/pylibcugraph/pylibcugraph/__init__.py @@ -16,7 +16,12 @@ weakly_connected_components, ) -# Import gpu_graph_data to add objects to experimental. -# Once moved out of experimental, import individual objects here to include in -# this namespace. -import pylibcugraph.structure.gpu_graph_data +# Mark certain cython APIs as experimental +# Pure python objects would use the @experimental decorator, but cython does not +# support decorators so it must be done here. +from pylibcugraph.utilities.experimental import experimental as __experimental + +from pylibcugraph._cugraph_c.sg_graph import SGGraph as __SGGraph +__experimental(ns_name="")(__SGGraph) +from pylibcugraph._cugraph_c.resource_handle import ResourceHandle as __ResourceHandle +__experimental(ns_name="")(__ResourceHandle) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd similarity index 94% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd index 0a1a5ffda45..2a7c4524c05 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd @@ -11,15 +11,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c._cugraph_api cimport ( +from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, cugraph_resource_handle_t, ) -from pylibcugraph._cugraph_c._error cimport ( +from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, cugraph_error_t, ) -from pylibcugraph._cugraph_c._array cimport ( +from pylibcugraph._cugraph_c.array cimport ( cugraph_type_erased_device_array_t, ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd similarity index 96% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd index 0366a94655e..6239f9f174c 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd @@ -11,11 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c._error cimport ( +from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, cugraph_error_t, ) -from pylibcugraph._cugraph_c._cugraph_api cimport ( +from pylibcugraph._cugraph_c.cugraph_api cimport ( cugraph_resource_handle_t, data_type_id_t, byte_t, diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd similarity index 100% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/error.pxd similarity index 100% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/error.pxd diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd similarity index 94% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd index 3dc3bfdc0d9..97af3ba93d2 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd @@ -11,15 +11,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c._cugraph_api cimport ( +from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, cugraph_resource_handle_t, ) -from pylibcugraph._cugraph_c._error cimport ( +from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, cugraph_error_t, ) -from pylibcugraph._cugraph_c._array cimport ( +from pylibcugraph._cugraph_c.array cimport ( cugraph_type_erased_device_array_t, cugraph_type_erased_host_array_t, ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_resource_handle.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx similarity index 95% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/_resource_handle.pyx rename to python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx index 052dd40e68b..1a644eb440b 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_resource_handle.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c._cugraph_api cimport ( +from pylibcugraph._cugraph_c.cugraph_api cimport ( cugraph_resource_handle_t, cugraph_create_resource_handle, cugraph_free_resource_handle, diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_sg_graph.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx similarity index 84% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/_sg_graph.pyx rename to python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx index 1f4fcda268f..e6f4b961090 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_sg_graph.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx @@ -13,18 +13,19 @@ from libc.stdint cimport uintptr_t -from pylibcugraph._cugraph_c._cugraph_api cimport ( +from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, cugraph_resource_handle_t, ) -from pylibcugraph._cugraph_c._error cimport ( +from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, cugraph_error_t, ) -from pylibcugraph._cugraph_c._array cimport ( +from pylibcugraph._cugraph_c.array cimport ( cugraph_type_erased_device_array_t, ) -from pylibcugraph._cugraph_c._graph cimport ( +from pylibcugraph._cugraph_c.graph cimport ( + cugraph_graph_t, cugraph_sg_graph_create, cugraph_graph_properties_t, cugraph_sg_graph_free, @@ -51,7 +52,7 @@ cdef class SGGraph: cdef cugraph_resource_handle_t* rh cdef cugraph_graph_properties_t* p cdef cugraph_type_erased_device_array_t* a - cdef cugraph_graph_t* self.__sg_graph + cdef cugraph_graph_t* g cdef cugraph_error_t* e cdef bool_t true=bool_t.TRUE cdef bool_t false=bool_t.FALSE @@ -66,6 +67,9 @@ cdef class SGGraph: false, &g, &e) + #self.__sg_graph = g def __dealloc__(self): - cugraph_sg_graph_free(self.__sg_graph) + #cdef cugraph_graph_t* g = self.__sg_graph + cdef cugraph_graph_t* g + cugraph_sg_graph_free(g) diff --git a/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py b/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py deleted file mode 100644 index 086c83f14f8..00000000000 --- a/python/pylibcugraph/pylibcugraph/structure/gpu_graph_data.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pylibcugraph.utilities.experimental import experimental -from pylibcugraph._cugraph_c._graph import cugraph_sg_graph_create -from pylibcugraph._cugraph_c._resource_handle import ResourceHandle - - -@experimental(ns_name="") -class __GPUGraphData: - """ - Data that resides on one or more GPUs that represents a graph. - - GPUGraphData instances define the mapping between user-provided data that - represents a graph (vertex IDs, edge lists) and the corresponding - representation needed to support GPU-based graph operations (0-based vertex - array indices, CSR, CSC, etc.). - - Deleting a GPUGraphData intance frees the GPU memory used for storing the - graph, but does not delete the src, dst, and weights arrays used to - construct the instance. - """ - - def __init__(self, src_array, dst_array, weight_array, - store_transposed=False, - resource_handle=None): - self.__graph = None - if resource_handle is None: - self.__resource_handle = ResourceHandle() - else: - self.__resource_handle = resource_handle - - cugraph_sg_graph_create(self.__resource_handle, - None,None,None,None,None,None,None) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_gpu_graph_data.py b/python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py similarity index 88% rename from python/pylibcugraph/pylibcugraph/tests/test_gpu_graph_data.py rename to python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py index 1f96d16fa36..7b343511a9b 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_gpu_graph_data.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py @@ -47,13 +47,13 @@ def graph_arrays(request): ############################################################################### # Tests def test_ctor(graph_arrays): - from pylibcugraph.experimental import GPUGraphData + from pylibcugraph.experimental import SGGraph (device_srcs, device_dsts, device_weights) = graph_arrays - G = GPUGraphData(src_array=device_srcs, - dst_array=device_dsts, - weight_array=device_weights, - store_transposed=False) + G = SGGraph(src_array=device_srcs, + dst_array=device_dsts, + weight_array=device_weights, + store_transposed=False) # FIXME: test for correct num verts, edges, etc. diff --git a/python/pylibcugraph/pylibcugraph/utilities/experimental.py b/python/pylibcugraph/pylibcugraph/utilities/experimental.py index 439abc7b3aa..b7ae7adf142 100644 --- a/python/pylibcugraph/pylibcugraph/utilities/experimental.py +++ b/python/pylibcugraph/pylibcugraph/utilities/experimental.py @@ -151,12 +151,16 @@ def experimental_ns_updater(obj): new_obj_name=obj.__name__.lstrip("__")) # Wrap the function in a function that prints a warning before # calling the obj. This is done after adding obj to the - # experimental namespace so the warning message would have the + # experimental namespace so the warning message will have the # properly-generated experimental names. warning_msg = (f"{new_ns_name}.{new_obj_name} is experimental and " "will change in a future release.") - obj.__module__ = new_ns_name - obj.__qualname__ = new_obj_name + # built-in/extension types cannot have these attrs set + try: + obj.__module__ = new_ns_name + obj.__qualname__ = new_obj_name + except TypeError: + pass @functools.wraps(obj) def call_with_warning(*args, **kwargs): warnings.warn(warning_msg, PendingDeprecationWarning) @@ -178,12 +182,16 @@ def call_with_warning(*args, **kwargs): new_obj_name=obj.__name__.lstrip("__")) # Wrap the function in a function that prints a warning before calling # the obj. This is done after adding obj to the experimental namespace - # so the warning message would have the properly-generated experimental + # so the warning message will have the properly-generated experimental # names. warning_msg = (f"{new_ns_name}.{new_obj_name} is experimental and " "will change in a future release.") - obj.__module__ = new_ns_name - obj.__qualname__ = new_obj_name + # built-in/extension types cannot have these attrs set + try: + obj.__module__ = new_ns_name + obj.__qualname__ = new_obj_name + except TypeError: + pass @functools.wraps(obj) def call_with_warning(*args, **kwargs): warnings.warn(warning_msg, PendingDeprecationWarning) From 2ea5408526fc6d6a9561648c2664bcf857548687 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 18 Jan 2022 20:59:50 -0600 Subject: [PATCH 06/43] Refactored experimental() to use a separate function for returning the callable, removed __init__.py file contents from connectivity in favor of just exposing functions in the top-level __init__.py. --- python/pylibcugraph/pylibcugraph/__init__.py | 10 +- .../_cugraph_c/resource_handle.pyx | 3 +- .../pylibcugraph/_cugraph_c/sg_graph.pyx | 3 +- .../pylibcugraph/components/__init__.py | 17 -- .../pylibcugraph/utilities/experimental.py | 192 +++++++++--------- 5 files changed, 109 insertions(+), 116 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/__init__.py b/python/pylibcugraph/pylibcugraph/__init__.py index 5383a66e48e..ac475dce3ca 100644 --- a/python/pylibcugraph/pylibcugraph/__init__.py +++ b/python/pylibcugraph/pylibcugraph/__init__.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph.components import ( +from pylibcugraph.components._connectivity import ( strongly_connected_components, weakly_connected_components, ) @@ -21,7 +21,7 @@ # support decorators so it must be done here. from pylibcugraph.utilities.experimental import experimental as __experimental -from pylibcugraph._cugraph_c.sg_graph import SGGraph as __SGGraph -__experimental(ns_name="")(__SGGraph) -from pylibcugraph._cugraph_c.resource_handle import ResourceHandle as __ResourceHandle -__experimental(ns_name="")(__ResourceHandle) +from pylibcugraph._cugraph_c.sg_graph import __SGGraph +__experimental(sub_ns_name="")(__SGGraph) +from pylibcugraph._cugraph_c.resource_handle import __ResourceHandle +__experimental(sub_ns_name="")(__ResourceHandle) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx index 1a644eb440b..01fa22d528a 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx @@ -18,7 +18,8 @@ from pylibcugraph._cugraph_c.cugraph_api cimport ( ) -cdef class ResourceHandle: +# FIXME: remove leading __ when no longer experimental +cdef class __ResourceHandle: """ RAII-stye resource handle class to manage individual create/free calls and the corresponding pointer to a cugraph_resource_handle_t diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx index e6f4b961090..7cf6d60cbbc 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx @@ -32,7 +32,8 @@ from pylibcugraph._cugraph_c.graph cimport ( ) -cdef class SGGraph: +# FIXME: remove leading __ when no longer experimental +cdef class __SGGraph: """ RAII-stye Graph class for use with single-GPU APIs that manages the individual create/free calls and the corresponding cugraph_graph_t pointer. diff --git a/python/pylibcugraph/pylibcugraph/components/__init__.py b/python/pylibcugraph/pylibcugraph/components/__init__.py index b4d9daae59e..e69de29bb2d 100644 --- a/python/pylibcugraph/pylibcugraph/components/__init__.py +++ b/python/pylibcugraph/pylibcugraph/components/__init__.py @@ -1,17 +0,0 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pylibcugraph.components._connectivity import ( - strongly_connected_components, - weakly_connected_components, -) diff --git a/python/pylibcugraph/pylibcugraph/utilities/experimental.py b/python/pylibcugraph/pylibcugraph/utilities/experimental.py index b7ae7adf142..dd2b2283db9 100644 --- a/python/pylibcugraph/pylibcugraph/utilities/experimental.py +++ b/python/pylibcugraph/pylibcugraph/utilities/experimental.py @@ -19,38 +19,43 @@ import pylibcugraph def add_obj_to_pylibcugraph_subnamespace(obj, sub_ns_name, - sub_sub_ns_name=None, - new_obj_name=None): + sub_sub_ns_name=None, + new_obj_name=None): """ - Adds an obj to the pylibcugraph. namespace, using the - objects current namespace names under pylibcugraph as the default - sub_sub_ns_name. + Adds an obj to the pylibcugraph.. namespace, + using the objects current namespace names under pylibcugraph as the default + sub_sub_ns_name, creating the sub-namespace and sub-sub-namespaces if + necessary. Example: - for pylibcugraph.link_analysis.pagerank.pagerank: - add_obj_to_pylibcugraph_subnamespace(pagerank, "experimental") + for object foo in pylibcugraph.structure + add_obj_to_pylibcugraph_subnamespace(foo, "experimental") results in: - pylibcugraph.experimental.link_analysis.pagerank.pagerank + pylibcugraph.experimental.structure.foo All namespaces - current and new - must be under "pylibcugraph". If sub_sub_ns_name is provided, it will be used to override the obj's current namespace under pylibcugraph. Example: - for pylibcugraph.link_analysis.pagerank.pagerank: - add_obj_to_pylibcugraph_subnamespace(pagerank, "experimental", - sub_sub_ns_name="foo.bar") + for object foo in pylibcugraph.structure + add_obj_to_pylibcugraph_subnamespace(foo, "experimental", + sub_sub_ns_name="bar.baz") results in: - pylibcugraph.experimental.foo.bar.pagerank + pylibcugraph.experimental.bar.baz.foo + + for object foo in pylibcugraph.structure + add_obj_to_pylibcugraph_subnamespace(foo, "experimental", + sub_sub_ns_name="") + results in: + pylibcugraph.experimental.foo - If new_obj_name is provided, it will be used to renamed the obj in the new - namespace. Example: - for pylibcugraph.link_analysis.pagerank.pagerank: - add_obj_to_pylibcugraph_subnamespace(pagerank, "experimental", - new_obj_name="new_pagerank") + for object foo in pylibcugraph.structure: + add_obj_to_pylibcugraph_subnamespace(foo, "experimental", + new_obj_name="new_pagerank") results in: - pylibcugraph.experimental.link_analysis.pagerank.new_pagerank + pylibcugraph.experimental.structure.new_pagerank Returns a tuple of: (new namespace name, new obj name, new namespace module obj) @@ -90,6 +95,49 @@ def add_obj_to_pylibcugraph_subnamespace(obj, sub_ns_name, mod_to_update) +def get_callable_for_experimental(sub_namespace_name=None): + """ + Returns a callable which can be used as the return value for the + "experimental" decorator function, or as something which can be called + directly. Calling the returned callable with an object as the arg results + in the object being added to the "experimental" namespace as described in + the docstring for the experimental decorator function. + + If sub_namespace_name is provided, the returned callable will add the object + to the sub namespace under experimental as described in the docstring for + the experimental decorator function. + """ + def experimental_ns_updater(obj): + (new_ns_name, new_obj_name, new_ns) = \ + add_obj_to_pylibcugraph_subnamespace( + obj, + sub_ns_name="experimental", + sub_sub_ns_name=sub_namespace_name, + new_obj_name=obj.__name__.lstrip("__")) + # Wrap the function in a function that prints a warning before + # calling the obj. This is done after adding obj to the + # experimental namespace so the warning message will have the + # properly-generated experimental names. + warning_msg = (f"{new_ns_name}.{new_obj_name} is experimental and " + "will change in a future release.") + # built-in/extension types cannot have these attrs set + try: + obj.__module__ = new_ns_name + obj.__qualname__ = new_obj_name + except TypeError: + pass + @functools.wraps(obj) + def call_with_warning(*args, **kwargs): + warnings.warn(warning_msg, PendingDeprecationWarning) + return obj(*args, **kwargs) + + wrapped_obj = call_with_warning + # Replace obj in the experimental ns with the wrapped obj + setattr(new_ns, new_obj_name, wrapped_obj) + return obj + return experimental_ns_updater + + def experimental(*args, **kwargs): """ Decorator function to add an obj to the pylibcugraph.experimental @@ -99,105 +147,65 @@ def experimental(*args, **kwargs): pylibcugraph.experimental..obj. Example: - for the pagerank function in pylibcugraph.link_analysis.pagerank: + for the foo function in pylibcugraph.structure: @experimental - def pagerank(...) + def foo(...) results in: - a pagerank() function in the - pylibcugraph.experimental.link_analysis.pagerank namespace. + a foo() function in the + pylibcugraph.experimental.structure namespace. - If the ns_name kwarg is given, it is used to replace the default + If the sub_ns_name kwarg is given, it is used to replace the default subnamespace under pylibcugraph that the obj currently resides in. Example: - for the pagerank function in pylibcugraph.link_analysis.pagerank: - @experimental(ns_name="foo") - def pagerank(...) + for the foo function in pylibcugraph.structure: + @experimental(sub_ns_name="bar") + def foo(...) results in: - a pagerank() function in the - pylibcugraph.experimental.foo namespace. + a foo() function in the + pylibcugraph.experimental.bar namespace. - Setting ns_name="" results in the obj being added directly to the - pylibcugraph.experimental namespace. + for the foo function in pylibcugraph.structure: + @experimental(sub_ns_name="") + def foo(...) + results in: + a foo() function added directly to + pylibcugraph.experimental If the current obj is private by naming it with a leading __, the leading __ is removed from the obj in the new namespace. This allows an experimental class/function to be private (hidden) in a non-experimental namespace but public in experimental. Example: - for the __pagerank function in pylibcugraph.link_analysis.pagerank: - @experimental(ns_name="foo") + for the __foo function in pylibcugraph.structure: + @experimental(sub_ns_name="bar") def __pagerank(...) results in: - a pagerank() function in the - pylibcugraph.experimental.foo namespace. + a foo() function in the + pylibcugraph.experimental.bar namespace. """ kwa = list(kwargs.keys()) - if kwa and kwa != ["ns_name"]: - raise TypeError("Only the 'ns_name' kwarg is allowed for " + if kwa and kwa != ["sub_ns_name"]: + raise TypeError("Only the 'sub_ns_name' kwarg is allowed for " "experimental()") - ns_name = kwargs.get("ns_name") + sub_ns_name = kwargs.get("sub_ns_name") # python expects decorators to return function wrappers one of two ways: if # args specified, the decorator must return a callable that accepts an obj # to wrap. If no args, then the decorator returns the wrapped obj directly. - if ns_name is not None: # called as @experimental(ns_name="...") - sub_namespace_name = ns_name - # The function to return that takes the obj and sets up the new ns - def experimental_ns_updater(obj): - (new_ns_name, new_obj_name, new_ns) = \ - add_obj_to_pylibcugraph_subnamespace( - obj, "experimental", - sub_sub_ns_name=sub_namespace_name, - new_obj_name=obj.__name__.lstrip("__")) - # Wrap the function in a function that prints a warning before - # calling the obj. This is done after adding obj to the - # experimental namespace so the warning message will have the - # properly-generated experimental names. - warning_msg = (f"{new_ns_name}.{new_obj_name} is experimental and " - "will change in a future release.") - # built-in/extension types cannot have these attrs set - try: - obj.__module__ = new_ns_name - obj.__qualname__ = new_obj_name - except TypeError: - pass - @functools.wraps(obj) - def call_with_warning(*args, **kwargs): - warnings.warn(warning_msg, PendingDeprecationWarning) - return obj(*args, **kwargs) - - wrapped_obj = call_with_warning - # Replace obj in the experimental ns with the wrapped obj - setattr(new_ns, new_obj_name, wrapped_obj) - return obj - return experimental_ns_updater + + if sub_ns_name is not None: # called as @experimental(sub_ns_name="...") + # Python will call the callable being returned here, which will then + # setup the new ns, add the obj to it, and return the original obj + return get_callable_for_experimental(sub_ns_name) else: # called as @experimental if len(args) > 1: raise TypeError("Too many positional args to experimental()") obj = args[0] - (new_ns_name, new_obj_name, new_ns) = \ - add_obj_to_pylibcugraph_subnamespace( - obj, "experimental", - new_obj_name=obj.__name__.lstrip("__")) - # Wrap the function in a function that prints a warning before calling - # the obj. This is done after adding obj to the experimental namespace - # so the warning message will have the properly-generated experimental - # names. - warning_msg = (f"{new_ns_name}.{new_obj_name} is experimental and " - "will change in a future release.") - # built-in/extension types cannot have these attrs set - try: - obj.__module__ = new_ns_name - obj.__qualname__ = new_obj_name - except TypeError: - pass - @functools.wraps(obj) - def call_with_warning(*args, **kwargs): - warnings.warn(warning_msg, PendingDeprecationWarning) - return obj(*args, **kwargs) - - wrapped_obj = call_with_warning - # Replace obj in the experimental ns with the wrapped obj - setattr(new_ns, new_obj_name, wrapped_obj) + # Get a callable to update the experimental namespace and call it + # directly here. + update_experimental = get_callable_for_experimental() + update_experimental(obj) + # Return the obj as-is, no need to return anything wrapped since + # update_experimental() did all the work. return obj From 7420e84dbeda7f84303cfef031741b0581ecf647 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Wed, 19 Jan 2022 05:38:40 -0600 Subject: [PATCH 07/43] Removed experimental decorator in favor of simple wrapper function and an experimental/__init__.py file. --- python/pylibcugraph/pylibcugraph/__init__.py | 12 +- .../pylibcugraph/experimental/__init__.py | 20 ++ .../pylibcugraph/utilities/api_tools.py | 48 ++++ .../pylibcugraph/utilities/experimental.py | 211 ------------------ 4 files changed, 70 insertions(+), 221 deletions(-) create mode 100644 python/pylibcugraph/pylibcugraph/experimental/__init__.py create mode 100644 python/pylibcugraph/pylibcugraph/utilities/api_tools.py delete mode 100644 python/pylibcugraph/pylibcugraph/utilities/experimental.py diff --git a/python/pylibcugraph/pylibcugraph/__init__.py b/python/pylibcugraph/pylibcugraph/__init__.py index ac475dce3ca..cc76f196b2e 100644 --- a/python/pylibcugraph/pylibcugraph/__init__.py +++ b/python/pylibcugraph/pylibcugraph/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. +# Copyright (c) 2021-2022, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -16,12 +16,4 @@ weakly_connected_components, ) -# Mark certain cython APIs as experimental -# Pure python objects would use the @experimental decorator, but cython does not -# support decorators so it must be done here. -from pylibcugraph.utilities.experimental import experimental as __experimental - -from pylibcugraph._cugraph_c.sg_graph import __SGGraph -__experimental(sub_ns_name="")(__SGGraph) -from pylibcugraph._cugraph_c.resource_handle import __ResourceHandle -__experimental(sub_ns_name="")(__ResourceHandle) +from pylibcugraph import experimental diff --git a/python/pylibcugraph/pylibcugraph/experimental/__init__.py b/python/pylibcugraph/pylibcugraph/experimental/__init__.py new file mode 100644 index 00000000000..8bb7ad4c7e3 --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/experimental/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph.utilities.api_tools import experimental_warning_wrapper + +from pylibcugraph._cugraph_c.sg_graph import __SGGraph +SGGraph = experimental_warning_wrapper(__SGGraph) + +from pylibcugraph._cugraph_c.resource_handle import __ResourceHandle +ResourceHandle = experimental_warning_wrapper(__ResourceHandle) diff --git a/python/pylibcugraph/pylibcugraph/utilities/api_tools.py b/python/pylibcugraph/pylibcugraph/utilities/api_tools.py new file mode 100644 index 00000000000..dec4c51b11e --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/utilities/api_tools.py @@ -0,0 +1,48 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import warnings +import inspect + + +def experimental_warning_wrapper(obj, keep_leading_underscores=False): + """ + Return a callable obj wrapped in a callable the prints a warning about it + being "experimental" (subject to change or removal) prior to calling it and + returning its value. + """ + obj_name = obj.__qualname__ + if keep_leading_underscores is False: + obj_name = obj_name.lstrip("__") + + # Assume the caller of this function is the namespace containing the + # experimental obj and try to get its namespace name. Default to no + # namespace name if it could not be found. + call_stack = inspect.stack() + calling_frame = call_stack[1].frame + ns_name = calling_frame.f_locals.get("__name__") + if ns_name is not None: + ns_name += "." + else: + ns_name = "" + + warning_msg = (f"{ns_name}{obj_name} is experimental and will change " + "or be removed in a future release.") + + @functools.wraps(obj) + def callable_warning_wrapper(*args, **kwargs): + warnings.warn(warning_msg, PendingDeprecationWarning) + return obj(*args, **kwargs) + + return callable_warning_wrapper diff --git a/python/pylibcugraph/pylibcugraph/utilities/experimental.py b/python/pylibcugraph/pylibcugraph/utilities/experimental.py deleted file mode 100644 index dd2b2283db9..00000000000 --- a/python/pylibcugraph/pylibcugraph/utilities/experimental.py +++ /dev/null @@ -1,211 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import functools -import types -import warnings -import sys - -import pylibcugraph - -def add_obj_to_pylibcugraph_subnamespace(obj, sub_ns_name, - sub_sub_ns_name=None, - new_obj_name=None): - """ - Adds an obj to the pylibcugraph.. namespace, - using the objects current namespace names under pylibcugraph as the default - sub_sub_ns_name, creating the sub-namespace and sub-sub-namespaces if - necessary. - - Example: - for object foo in pylibcugraph.structure - add_obj_to_pylibcugraph_subnamespace(foo, "experimental") - results in: - pylibcugraph.experimental.structure.foo - - All namespaces - current and new - must be under "pylibcugraph". - - If sub_sub_ns_name is provided, it will be used to override the obj's - current namespace under pylibcugraph. - Example: - for object foo in pylibcugraph.structure - add_obj_to_pylibcugraph_subnamespace(foo, "experimental", - sub_sub_ns_name="bar.baz") - results in: - pylibcugraph.experimental.bar.baz.foo - - for object foo in pylibcugraph.structure - add_obj_to_pylibcugraph_subnamespace(foo, "experimental", - sub_sub_ns_name="") - results in: - pylibcugraph.experimental.foo - - Example: - for object foo in pylibcugraph.structure: - add_obj_to_pylibcugraph_subnamespace(foo, "experimental", - new_obj_name="new_pagerank") - results in: - pylibcugraph.experimental.structure.new_pagerank - - Returns a tuple of: - (new namespace name, new obj name, new namespace module obj) - """ - # Create a list of names representing the ns to create - current_mod_name_parts = obj.__module__.split(".") - # All namespaces - current and new - must be under "pylibcugraph" - if current_mod_name_parts[0] != "pylibcugraph": - raise ValueError(f"{obj.__name__} is not under the pylibcugraph " - "package") - new_mod_name_parts = [sub_ns_name] - if sub_sub_ns_name is None: - new_mod_name_parts += current_mod_name_parts[1:] - else: - if sub_sub_ns_name != "": - new_mod_name_parts += sub_sub_ns_name.split(".") - - # Create the new namespace - mod_to_update = pylibcugraph - mod_name_parts = ["pylibcugraph"] - - for ns in new_mod_name_parts: - mod_name_parts.append(ns) - if not(hasattr(mod_to_update, ns)): - mod_to_update_name = ".".join(mod_name_parts) - new_mod = types.ModuleType(mod_to_update_name) - setattr(mod_to_update, ns, new_mod) - sys.modules[mod_to_update_name] = new_mod - mod_to_update = getattr(mod_to_update, ns) - - # Add obj to the new namespace - new_obj_name = obj.__name__ if new_obj_name is None else new_obj_name - setattr(mod_to_update, new_obj_name, obj) - - return (".".join(["pylibcugraph"] + new_mod_name_parts), - new_obj_name, - mod_to_update) - - -def get_callable_for_experimental(sub_namespace_name=None): - """ - Returns a callable which can be used as the return value for the - "experimental" decorator function, or as something which can be called - directly. Calling the returned callable with an object as the arg results - in the object being added to the "experimental" namespace as described in - the docstring for the experimental decorator function. - - If sub_namespace_name is provided, the returned callable will add the object - to the sub namespace under experimental as described in the docstring for - the experimental decorator function. - """ - def experimental_ns_updater(obj): - (new_ns_name, new_obj_name, new_ns) = \ - add_obj_to_pylibcugraph_subnamespace( - obj, - sub_ns_name="experimental", - sub_sub_ns_name=sub_namespace_name, - new_obj_name=obj.__name__.lstrip("__")) - # Wrap the function in a function that prints a warning before - # calling the obj. This is done after adding obj to the - # experimental namespace so the warning message will have the - # properly-generated experimental names. - warning_msg = (f"{new_ns_name}.{new_obj_name} is experimental and " - "will change in a future release.") - # built-in/extension types cannot have these attrs set - try: - obj.__module__ = new_ns_name - obj.__qualname__ = new_obj_name - except TypeError: - pass - @functools.wraps(obj) - def call_with_warning(*args, **kwargs): - warnings.warn(warning_msg, PendingDeprecationWarning) - return obj(*args, **kwargs) - - wrapped_obj = call_with_warning - # Replace obj in the experimental ns with the wrapped obj - setattr(new_ns, new_obj_name, wrapped_obj) - return obj - return experimental_ns_updater - - -def experimental(*args, **kwargs): - """ - Decorator function to add an obj to the pylibcugraph.experimental - namespace. - - If no args are given, obj is copied to - pylibcugraph.experimental..obj. - - Example: - for the foo function in pylibcugraph.structure: - @experimental - def foo(...) - results in: - a foo() function in the - pylibcugraph.experimental.structure namespace. - - If the sub_ns_name kwarg is given, it is used to replace the default - subnamespace under pylibcugraph that the obj currently resides in. - Example: - for the foo function in pylibcugraph.structure: - @experimental(sub_ns_name="bar") - def foo(...) - results in: - a foo() function in the - pylibcugraph.experimental.bar namespace. - - for the foo function in pylibcugraph.structure: - @experimental(sub_ns_name="") - def foo(...) - results in: - a foo() function added directly to - pylibcugraph.experimental - - If the current obj is private by naming it with a leading __, the leading - __ is removed from the obj in the new namespace. This allows an - experimental class/function to be private (hidden) in a non-experimental - namespace but public in experimental. - Example: - for the __foo function in pylibcugraph.structure: - @experimental(sub_ns_name="bar") - def __pagerank(...) - results in: - a foo() function in the - pylibcugraph.experimental.bar namespace. - """ - kwa = list(kwargs.keys()) - if kwa and kwa != ["sub_ns_name"]: - raise TypeError("Only the 'sub_ns_name' kwarg is allowed for " - "experimental()") - sub_ns_name = kwargs.get("sub_ns_name") - - # python expects decorators to return function wrappers one of two ways: if - # args specified, the decorator must return a callable that accepts an obj - # to wrap. If no args, then the decorator returns the wrapped obj directly. - - if sub_ns_name is not None: # called as @experimental(sub_ns_name="...") - # Python will call the callable being returned here, which will then - # setup the new ns, add the obj to it, and return the original obj - return get_callable_for_experimental(sub_ns_name) - - else: # called as @experimental - if len(args) > 1: - raise TypeError("Too many positional args to experimental()") - obj = args[0] - # Get a callable to update the experimental namespace and call it - # directly here. - update_experimental = get_callable_for_experimental() - update_experimental(obj) - # Return the obj as-is, no need to return anything wrapped since - # update_experimental() did all the work. - return obj From da468a4fc2fe706dce30b24efe5962b8eb47483e Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Wed, 19 Jan 2022 22:00:24 -0600 Subject: [PATCH 08/43] More progress on SGGraph (just need C API to do device-to-device array copies), added additional types for C API structs, added tests. --- .../_cugraph_c/graph_properties.pxd | 20 +++ .../_cugraph_c/graph_properties.pyx | 46 +++++++ .../_cugraph_c/resource_handle.pxd | 20 +++ .../_cugraph_c/resource_handle.pyx | 10 +- .../pylibcugraph/_cugraph_c/sg_graph.pxd | 20 +++ .../pylibcugraph/_cugraph_c/sg_graph.pyx | 126 ++++++++++++++---- .../pylibcugraph/experimental/__init__.py | 11 +- .../pylibcugraph/tests/test__cugraph_c.py | 72 ++++++++++ .../pylibcugraph/utilities/api_tools.py | 20 ++- 9 files changed, 300 insertions(+), 45 deletions(-) create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pxd create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pyx create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pxd create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pxd create mode 100644 python/pylibcugraph/pylibcugraph/tests/test__cugraph_c.py diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pxd new file mode 100644 index 00000000000..37511503360 --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pxd @@ -0,0 +1,20 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph._cugraph_c.graph cimport ( + cugraph_graph_properties_t, +) + + +cdef class EXPERIMENTAL__GraphProperties: + cdef cugraph_graph_properties_t c_graph_properties diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pyx new file mode 100644 index 00000000000..e0c6a6af77d --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pyx @@ -0,0 +1,46 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph._cugraph_c.cugraph_api cimport ( + bool_t, +) + + +cdef class EXPERIMENTAL__GraphProperties: + """ + """ + def __cinit__(self): + self.c_graph_properties.is_symmetric = bool_t.FALSE + self.c_graph_properties.is_multigraph = bool_t.FALSE + + @property + def is_symmetric(self): + return bool(self.c_graph_properties.is_symmetric) + + @is_symmetric.setter + def is_symmetric(self, value): + if not(isinstance(value, (int, bool))): + raise TypeError(f"expected int or bool, got {type(value)}") + + self.c_graph_properties.is_symmetric = int(value) + + @property + def is_multigraph(self): + return bool(self.c_graph_properties.is_multigraph) + + @is_multigraph.setter + def is_multigraph(self, value): + if not(isinstance(value, (int, bool))): + raise TypeError(f"expected int or bool, got {type(value)}") + + self.c_graph_properties.is_multigraph = int(value) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pxd new file mode 100644 index 00000000000..afe6ba18a27 --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pxd @@ -0,0 +1,20 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph._cugraph_c.cugraph_api cimport ( + cugraph_resource_handle_t, +) + + +cdef class EXPERIMENTAL__ResourceHandle: + cdef cugraph_resource_handle_t* c_resource_handle_ptr diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx index 01fa22d528a..30fc62ffd6d 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx @@ -12,24 +12,20 @@ # limitations under the License. from pylibcugraph._cugraph_c.cugraph_api cimport ( - cugraph_resource_handle_t, cugraph_create_resource_handle, cugraph_free_resource_handle, ) -# FIXME: remove leading __ when no longer experimental -cdef class __ResourceHandle: +cdef class EXPERIMENTAL__ResourceHandle: """ RAII-stye resource handle class to manage individual create/free calls and the corresponding pointer to a cugraph_resource_handle_t """ - cdef cugraph_resource_handle_t* __handle - def __cinit__(self): - self.__handle = cugraph_create_resource_handle() + self.c_resource_handle_ptr = cugraph_create_resource_handle() # FIXME: check for error def __dealloc__(self): # FIXME: free only if handle is a valid pointer - cugraph_free_resource_handle(self.__handle) + cugraph_free_resource_handle(self.c_resource_handle_ptr) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pxd new file mode 100644 index 00000000000..92b6c74ee94 --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pxd @@ -0,0 +1,20 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph._cugraph_c.graph cimport ( + cugraph_graph_t, +) + + +cdef class EXPERIMENTAL__SGGraph: + cdef cugraph_graph_t* c_sg_graph_ptr diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx index 7cf6d60cbbc..b0ff7d1b46f 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx @@ -16,6 +16,7 @@ from libc.stdint cimport uintptr_t from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, cugraph_resource_handle_t, + data_type_id_t, ) from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, @@ -23,6 +24,8 @@ from pylibcugraph._cugraph_c.error cimport ( ) from pylibcugraph._cugraph_c.array cimport ( cugraph_type_erased_device_array_t, + cugraph_type_erased_device_array_create, + cugraph_type_erased_device_array_free, ) from pylibcugraph._cugraph_c.graph cimport ( cugraph_graph_t, @@ -30,47 +33,112 @@ from pylibcugraph._cugraph_c.graph cimport ( cugraph_graph_properties_t, cugraph_sg_graph_free, ) +from pylibcugraph._cugraph_c.resource_handle cimport ( + EXPERIMENTAL__ResourceHandle, +) +from pylibcugraph._cugraph_c.graph_properties cimport ( + EXPERIMENTAL__GraphProperties, +) + + +# FIXME: add tests for this +cdef assert_success(cugraph_error_code_t code, + cugraph_error_t* err, + api_name): + if code != cugraph_error_code_t.CUGRAPH_SUCCESS: + # FIXME: extract message using cugraph_error_message() + raise RuntimeError(f"non-success value returned from {api_name}") -# FIXME: remove leading __ when no longer experimental -cdef class __SGGraph: +cdef class EXPERIMENTAL__SGGraph: """ RAII-stye Graph class for use with single-GPU APIs that manages the individual create/free calls and the corresponding cugraph_graph_t pointer. """ def __cinit__(self, - resource_handle, - graph_properties, + EXPERIMENTAL__ResourceHandle resource_handle, + EXPERIMENTAL__GraphProperties graph_properties, src_array, dst_array, weight_array, store_transposed, renumber, expensive_check): - # use __cuda_array_interface__ to get device pointers - - cdef uintptr_t x = 0; - cdef cugraph_resource_handle_t* rh - cdef cugraph_graph_properties_t* p - cdef cugraph_type_erased_device_array_t* a - cdef cugraph_graph_t* g - cdef cugraph_error_t* e - cdef bool_t true=bool_t.TRUE - cdef bool_t false=bool_t.FALSE - - cugraph_sg_graph_create(rh, - p, - a, - a, - a, - true, - false, - false, - &g, - &e) - #self.__sg_graph = g + + if not(isinstance(store_transposed, (int, bool))): + raise TypeError("expected int or bool for store_transposed, got " + f"{type(store_transposed)}") + if not(isinstance(renumber, (int, bool))): + raise TypeError("expected int or bool for renumber, got " + f"{type(renumber)}") + if not(isinstance(expensive_check, (int, bool))): + raise TypeError("expected int or bool for expensive_check, got " + f"{type(expensive_check)}") + if not(hasattr(src_array, "__cuda_array_interface__")): + raise TypeError("src_array does not have required " + "__cuda_array_interface__ attr") + if not(hasattr(dst_array, "__cuda_array_interface__")): + raise TypeError("dst_array does not have required " + "__cuda_array_interface__ attr") + if not(hasattr(weight_array, "__cuda_array_interface__")): + raise TypeError("weight_array does not have required " + "__cuda_array_interface__ attr") + + cdef cugraph_error_t* error_ptr + cdef cugraph_error_code_t err_code + + cdef cugraph_type_erased_device_array_t* srcs_ptr + cdef cugraph_type_erased_device_array_t* dsts_ptr + cdef cugraph_type_erased_device_array_t* weights_ptr + + # FIXME: set dtype properly + err_code = cugraph_type_erased_device_array_create( + resource_handle.c_resource_handle_ptr, + data_type_id_t.INT32, + len(src_array), + &srcs_ptr, + &error_ptr) + + assert_success(err_code, error_ptr, + "cugraph_type_erased_device_array_create()") + + # FIXME: set dtype properly + err_code = cugraph_type_erased_device_array_create( + resource_handle.c_resource_handle_ptr, + data_type_id_t.INT32, + len(dst_array), + &dsts_ptr, + &error_ptr) + + assert_success(err_code, error_ptr, + "cugraph_type_erased_device_array_create()") + + # FIXME: set dtype properly + err_code = cugraph_type_erased_device_array_create( + resource_handle.c_resource_handle_ptr, + data_type_id_t.INT32, + len(weight_array), + &weights_ptr, + &error_ptr) + + assert_success(err_code, error_ptr, + "cugraph_type_erased_device_array_create()") + + err_code = cugraph_sg_graph_create( + resource_handle.c_resource_handle_ptr, + &(graph_properties.c_graph_properties), + srcs_ptr, + dsts_ptr, + weights_ptr, + int(store_transposed), + int(renumber), + int(expensive_check), + &(self.c_sg_graph_ptr), + &error_ptr) + + assert_success(err_code, error_ptr, + "cugraph_sg_graph_create()") def __dealloc__(self): - #cdef cugraph_graph_t* g = self.__sg_graph - cdef cugraph_graph_t* g - cugraph_sg_graph_free(g) + if self.c_sg_graph_ptr is not NULL: + cugraph_sg_graph_free(self.c_sg_graph_ptr) diff --git a/python/pylibcugraph/pylibcugraph/experimental/__init__.py b/python/pylibcugraph/pylibcugraph/experimental/__init__.py index 8bb7ad4c7e3..72d21e92829 100644 --- a/python/pylibcugraph/pylibcugraph/experimental/__init__.py +++ b/python/pylibcugraph/pylibcugraph/experimental/__init__.py @@ -13,8 +13,11 @@ from pylibcugraph.utilities.api_tools import experimental_warning_wrapper -from pylibcugraph._cugraph_c.sg_graph import __SGGraph -SGGraph = experimental_warning_wrapper(__SGGraph) +from pylibcugraph._cugraph_c.sg_graph import EXPERIMENTAL__SGGraph +SGGraph = experimental_warning_wrapper(EXPERIMENTAL__SGGraph) -from pylibcugraph._cugraph_c.resource_handle import __ResourceHandle -ResourceHandle = experimental_warning_wrapper(__ResourceHandle) +from pylibcugraph._cugraph_c.resource_handle import EXPERIMENTAL__ResourceHandle +ResourceHandle = experimental_warning_wrapper(EXPERIMENTAL__ResourceHandle) + +from pylibcugraph._cugraph_c.graph_properties import EXPERIMENTAL__GraphProperties +GraphProperties = experimental_warning_wrapper(EXPERIMENTAL__GraphProperties) diff --git a/python/pylibcugraph/pylibcugraph/tests/test__cugraph_c.py b/python/pylibcugraph/pylibcugraph/tests/test__cugraph_c.py new file mode 100644 index 00000000000..442d13a854c --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/tests/test__cugraph_c.py @@ -0,0 +1,72 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import cupy as cp +import numpy as np + + +def test_graph_properties(): + from pylibcugraph.experimental import GraphProperties + + gp = GraphProperties() + assert gp.is_symmetric is False + assert gp.is_multigraph is False + + gp.is_symmetric = True + assert gp.is_symmetric is True + gp.is_symmetric = 0 + assert gp.is_symmetric is False + with pytest.raises(TypeError): + gp.is_symmetric = "foo" + + gp.is_multigraph = True + assert gp.is_multigraph is True + gp.is_multigraph = 0 + assert gp.is_multigraph is False + with pytest.raises(TypeError): + gp.is_multigraph = "foo" + + +def test_resource_handle(): + from pylibcugraph.experimental import ResourceHandle + # This type has no attributes and is just defined to pass a struct from C + # back in to C. In the future it may take args to acquire specific + # resources, but for now just make sure nothing crashes. + rh = ResourceHandle() + del rh + + +def test_sg_graph(): + from pylibcugraph.experimental import (SGGraph, + ResourceHandle, + GraphProperties, + ) + + graph_props = GraphProperties() + graph_props.is_symmetric = False + graph_props.is_multigraph = False + resource_handle = ResourceHandle() + + srcs = cp.asarray([0, 1, 2], dtype=np.int32) + dsts = cp.asarray([1, 2, 3], dtype=np.int32) + weights = cp.asarray([0, 0, 0, 0], dtype=np.int32) + + g = SGGraph(resource_handle, + graph_props, + srcs, + dsts, + weights, + store_transposed=False, + renumber=False, + expensive_check=False) diff --git a/python/pylibcugraph/pylibcugraph/utilities/api_tools.py b/python/pylibcugraph/pylibcugraph/utilities/api_tools.py index dec4c51b11e..e0281d86e5c 100644 --- a/python/pylibcugraph/pylibcugraph/utilities/api_tools.py +++ b/python/pylibcugraph/pylibcugraph/utilities/api_tools.py @@ -15,18 +15,28 @@ import warnings import inspect +experimental_prefix = "EXPERIMENTAL" -def experimental_warning_wrapper(obj, keep_leading_underscores=False): + +def experimental_warning_wrapper(obj, make_public_name=True): """ Return a callable obj wrapped in a callable the prints a warning about it - being "experimental" (subject to change or removal) prior to calling it and - returning its value. + being "experimental" (an object that is in the public API but subject to + change or removal) prior to calling it and returning its value. + + If make_public_name is False, the object's name used in the warning message + is left unmodified. If True (default), any leading __ and/or EXPERIMENTAL + string are removed from the name used in warning messages. This allows an + object to be named with a "private" name in the public API so it can remain + hidden while it is still experimental, but have a public name within the + experimental namespace so it can be easily discovered and used. """ obj_name = obj.__qualname__ - if keep_leading_underscores is False: + if make_public_name: + obj_name = obj_name.lstrip(experimental_prefix) obj_name = obj_name.lstrip("__") - # Assume the caller of this function is the namespace containing the + # Assume the caller of this function is the module containing the # experimental obj and try to get its namespace name. Default to no # namespace name if it could not be found. call_stack = inspect.stack() From 2c2f85b192d36884555fe7d5bb72472dfabdc410 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Thu, 20 Jan 2022 09:38:56 -0600 Subject: [PATCH 09/43] flake8 fixes. --- python/pylibcugraph/pylibcugraph/tests/test__cugraph_c.py | 4 +++- python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/python/pylibcugraph/pylibcugraph/tests/test__cugraph_c.py b/python/pylibcugraph/pylibcugraph/tests/test__cugraph_c.py index 442d13a854c..0db43d08e03 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test__cugraph_c.py +++ b/python/pylibcugraph/pylibcugraph/tests/test__cugraph_c.py @@ -51,7 +51,7 @@ def test_sg_graph(): from pylibcugraph.experimental import (SGGraph, ResourceHandle, GraphProperties, - ) + ) graph_props = GraphProperties() graph_props.is_symmetric = False @@ -70,3 +70,5 @@ def test_sg_graph(): store_transposed=False, renumber=False, expensive_check=False) + + print(g) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py b/python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py index 7b343511a9b..62f1c06bc03 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py @@ -27,6 +27,7 @@ utils.RAPIDS_DATASET_ROOT_DIR_PATH/"dolphins.csv", ] + @pytest.fixture(scope="module", params=[pytest.param(ds, id=ds.name) for ds in datasets]) def graph_arrays(request): @@ -56,4 +57,5 @@ def test_ctor(graph_arrays): weight_array=device_weights, store_transposed=False) + print(G) # FIXME: test for correct num verts, edges, etc. From ab387c1d022ea3308c31d06b9bcc101a545f63f8 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Thu, 20 Jan 2022 10:04:39 -0600 Subject: [PATCH 10/43] flake8 fix, updated copyright dates. --- python/pylibcugraph/pylibcugraph/tests/utils.py | 2 +- python/pylibcugraph/pylibcugraph/utilities/__init__.py | 1 - python/pylibcugraph/setup.py | 5 +++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/tests/utils.py b/python/pylibcugraph/pylibcugraph/tests/utils.py index 03ddcd3c534..bdf15b8efa4 100644 --- a/python/pylibcugraph/pylibcugraph/tests/utils.py +++ b/python/pylibcugraph/pylibcugraph/tests/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. +# Copyright (c) 2021-2022, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/pylibcugraph/pylibcugraph/utilities/__init__.py b/python/pylibcugraph/pylibcugraph/utilities/__init__.py index 8b137891791..e69de29bb2d 100644 --- a/python/pylibcugraph/pylibcugraph/utilities/__init__.py +++ b/python/pylibcugraph/pylibcugraph/utilities/__init__.py @@ -1 +0,0 @@ - diff --git a/python/pylibcugraph/setup.py b/python/pylibcugraph/setup.py index d4ce8fb6793..da8832b80ea 100644 --- a/python/pylibcugraph/setup.py +++ b/python/pylibcugraph/setup.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2021, NVIDIA CORPORATION. +# Copyright (c) 2018-2022, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -130,7 +130,8 @@ def run(self): cuda_lib_dir, os.path.join(os.sys.prefix, "lib") ], - libraries=['cudart', 'cusparse', 'cusolver', 'cugraph', 'nccl', 'cugraph_c'], + libraries=['cudart', 'cusparse', 'cusolver', 'cugraph', 'nccl', + 'cugraph_c'], language='c++', extra_compile_args=['-std=c++17']) ] From 6ad71c88d452261607c0005261bf07179b1fdae5 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Thu, 20 Jan 2022 15:10:18 -0600 Subject: [PATCH 11/43] Test refactoring. --- .../pylibcugraph/_cugraph_c/sg_graph.pyx | 21 +-- .../pylibcugraph/_cugraph_c/utils.pxd | 22 +++ .../pylibcugraph/_cugraph_c/utils.pyx | 21 +++ .../pylibcugraph/tests/test__cugraph_c.py | 74 ---------- .../pylibcugraph/tests/test_sg_graph.py | 131 +++++++++++++++--- 5 files changed, 166 insertions(+), 103 deletions(-) create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pxd create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pyx delete mode 100644 python/pylibcugraph/pylibcugraph/tests/test__cugraph_c.py diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx index b0ff7d1b46f..57be62bb47e 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx @@ -39,15 +39,9 @@ from pylibcugraph._cugraph_c.resource_handle cimport ( from pylibcugraph._cugraph_c.graph_properties cimport ( EXPERIMENTAL__GraphProperties, ) - - -# FIXME: add tests for this -cdef assert_success(cugraph_error_code_t code, - cugraph_error_t* err, - api_name): - if code != cugraph_error_code_t.CUGRAPH_SUCCESS: - # FIXME: extract message using cugraph_error_message() - raise RuntimeError(f"non-success value returned from {api_name}") +from pylibcugraph._cugraph_c.utils cimport ( + assert_success, +) cdef class EXPERIMENTAL__SGGraph: @@ -65,6 +59,7 @@ cdef class EXPERIMENTAL__SGGraph: renumber, expensive_check): + # FIXME: add tests for these if not(isinstance(store_transposed, (int, bool))): raise TypeError("expected int or bool for store_transposed, got " f"{type(store_transposed)}") @@ -101,6 +96,8 @@ cdef class EXPERIMENTAL__SGGraph: assert_success(err_code, error_ptr, "cugraph_type_erased_device_array_create()") + # FIXME: add call to to device-device copy of __cuda_array_interface__ + # values to cugraph_type_erased_device_array # FIXME: set dtype properly err_code = cugraph_type_erased_device_array_create( @@ -112,17 +109,21 @@ cdef class EXPERIMENTAL__SGGraph: assert_success(err_code, error_ptr, "cugraph_type_erased_device_array_create()") + # FIXME: add call to to device-device copy of __cuda_array_interface__ + # values to cugraph_type_erased_device_array # FIXME: set dtype properly err_code = cugraph_type_erased_device_array_create( resource_handle.c_resource_handle_ptr, - data_type_id_t.INT32, + data_type_id_t.FLOAT32, len(weight_array), &weights_ptr, &error_ptr) assert_success(err_code, error_ptr, "cugraph_type_erased_device_array_create()") + # FIXME: add call to to device-device copy of __cuda_array_interface__ + # values to cugraph_type_erased_device_array err_code = cugraph_sg_graph_create( resource_handle.c_resource_handle_ptr, diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pxd new file mode 100644 index 00000000000..1836196124b --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pxd @@ -0,0 +1,22 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph._cugraph_c.error cimport ( + cugraph_error_code_t, + cugraph_error_t, +) + + +cdef assert_success(cugraph_error_code_t code, + cugraph_error_t* err, + api_name) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pyx new file mode 100644 index 00000000000..b3bbbf7ef71 --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pyx @@ -0,0 +1,21 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# FIXME: add tests for this +cdef assert_success(cugraph_error_code_t code, + cugraph_error_t* err, + api_name): + if code != cugraph_error_code_t.CUGRAPH_SUCCESS: + # FIXME: extract message using cugraph_error_message() + raise RuntimeError(f"non-success value returned from {api_name}") diff --git a/python/pylibcugraph/pylibcugraph/tests/test__cugraph_c.py b/python/pylibcugraph/pylibcugraph/tests/test__cugraph_c.py deleted file mode 100644 index 0db43d08e03..00000000000 --- a/python/pylibcugraph/pylibcugraph/tests/test__cugraph_c.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -import cupy as cp -import numpy as np - - -def test_graph_properties(): - from pylibcugraph.experimental import GraphProperties - - gp = GraphProperties() - assert gp.is_symmetric is False - assert gp.is_multigraph is False - - gp.is_symmetric = True - assert gp.is_symmetric is True - gp.is_symmetric = 0 - assert gp.is_symmetric is False - with pytest.raises(TypeError): - gp.is_symmetric = "foo" - - gp.is_multigraph = True - assert gp.is_multigraph is True - gp.is_multigraph = 0 - assert gp.is_multigraph is False - with pytest.raises(TypeError): - gp.is_multigraph = "foo" - - -def test_resource_handle(): - from pylibcugraph.experimental import ResourceHandle - # This type has no attributes and is just defined to pass a struct from C - # back in to C. In the future it may take args to acquire specific - # resources, but for now just make sure nothing crashes. - rh = ResourceHandle() - del rh - - -def test_sg_graph(): - from pylibcugraph.experimental import (SGGraph, - ResourceHandle, - GraphProperties, - ) - - graph_props = GraphProperties() - graph_props.is_symmetric = False - graph_props.is_multigraph = False - resource_handle = ResourceHandle() - - srcs = cp.asarray([0, 1, 2], dtype=np.int32) - dsts = cp.asarray([1, 2, 3], dtype=np.int32) - weights = cp.asarray([0, 0, 0, 0], dtype=np.int32) - - g = SGGraph(resource_handle, - graph_props, - srcs, - dsts, - weights, - store_transposed=False, - renumber=False, - expensive_check=False) - - print(g) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py b/python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py index 62f1c06bc03..f334d74f842 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py @@ -23,39 +23,132 @@ # ============================================================================= # Pytest fixtures # ============================================================================= +class InlineGraphData: + @property + def name(self): + return self.__class__.__name__ + + @property + def is_valid(self): + return not(self.name.startswith("Invalid")) + +class InvalidNumWeights_1(InlineGraphData): # noqa: E302 + srcs = cp.asarray([0, 1, 2], dtype=np.int32) + dsts = cp.asarray([1, 2, 3], dtype=np.int32) + weights = cp.asarray([0, 0, 0, 0], dtype=np.int32) + +class InvalidNumVerts_1(InlineGraphData): # noqa: E302 + srcs = cp.asarray([1, 2], dtype=np.int32) + dsts = cp.asarray([1, 2, 3], dtype=np.int32) + weights = cp.asarray([0, 0, 0], dtype=np.int32) + +class Simple_1(InlineGraphData): # noqa: E302 + srcs = cp.asarray([0, 1, 2], dtype=np.int32) + dsts = cp.asarray([1, 2, 3], dtype=np.int32) + weights = cp.asarray([0, 0, 0], dtype=np.int32) + + datasets = [utils.RAPIDS_DATASET_ROOT_DIR_PATH/"karate.csv", utils.RAPIDS_DATASET_ROOT_DIR_PATH/"dolphins.csv", + InvalidNumWeights_1(), + InvalidNumVerts_1(), + Simple_1(), ] @pytest.fixture(scope="module", params=[pytest.param(ds, id=ds.name) for ds in datasets]) -def graph_arrays(request): +def graph_data(request): ds = request.param - pdf = pd.read_csv(ds, - delimiter=" ", header=None, - names=["0", "1", "weight"], - dtype={"0": "int32", "1": "int32", "weight": "float32"}, - ) - device_srcs = cp.asarray(pdf["0"].to_numpy(), dtype=np.int32) - device_dsts = cp.asarray(pdf["1"].to_numpy(), dtype=np.int32) - device_weights = cp.asarray(pdf["weight"].to_numpy(), dtype=np.float32) + if isinstance(ds, InlineGraphData): + device_srcs = ds.srcs + device_dsts = ds.dsts + device_weights = ds.weights + is_valid = ds.is_valid + else: + pdf = pd.read_csv(ds, + delimiter=" ", header=None, + names=["0", "1", "weight"], + dtype={"0": "int32", "1": "int32", + "weight": "float32"}, + ) + device_srcs = cp.asarray(pdf["0"].to_numpy(), dtype=np.int32) + device_dsts = cp.asarray(pdf["1"].to_numpy(), dtype=np.int32) + device_weights = cp.asarray(pdf["weight"].to_numpy(), dtype=np.float32) + # Assume all datasets on disk are valid + is_valid = True - return (device_srcs, device_dsts, device_weights) + return (device_srcs, device_dsts, device_weights, is_valid) ############################################################################### # Tests -def test_ctor(graph_arrays): - from pylibcugraph.experimental import SGGraph +def test_graph_properties(): + from pylibcugraph.experimental import GraphProperties + + gp = GraphProperties() + assert gp.is_symmetric is False + assert gp.is_multigraph is False + + gp.is_symmetric = True + assert gp.is_symmetric is True + gp.is_symmetric = 0 + assert gp.is_symmetric is False + with pytest.raises(TypeError): + gp.is_symmetric = "foo" + + gp.is_multigraph = True + assert gp.is_multigraph is True + gp.is_multigraph = 0 + assert gp.is_multigraph is False + with pytest.raises(TypeError): + gp.is_multigraph = "foo" + + +def test_resource_handle(): + from pylibcugraph.experimental import ResourceHandle + # This type has no attributes and is just defined to pass a struct from C + # back in to C. In the future it may take args to acquire specific + # resources, but for now just make sure nothing crashes. + rh = ResourceHandle() + del rh + + +def test_sg_graph_ctor(graph_data): + + from pylibcugraph.experimental import (SGGraph, + ResourceHandle, + GraphProperties, + ) + + (device_srcs, device_dsts, device_weights, is_valid) = graph_data + + graph_props = GraphProperties() + graph_props.is_symmetric = False + graph_props.is_multigraph = False + resource_handle = ResourceHandle() - (device_srcs, device_dsts, device_weights) = graph_arrays + if is_valid: + g = SGGraph(resource_handle, + graph_props, + device_srcs, + device_dsts, + device_weights, + store_transposed=False, + renumber=False, + expensive_check=False) - G = SGGraph(src_array=device_srcs, - dst_array=device_dsts, - weight_array=device_weights, - store_transposed=False) + print(g) + else: + with pytest.raises(RuntimeError): + g = SGGraph(resource_handle, + graph_props, + device_srcs, + device_dsts, + device_weights, + store_transposed=False, + renumber=False, + expensive_check=False) - print(G) - # FIXME: test for correct num verts, edges, etc. + print(g) From c35740ecf9eef1f28d4a6073252e8e76e60bbd0a Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Thu, 20 Jan 2022 20:30:34 -0600 Subject: [PATCH 12/43] Added start of pagerank bindings, renamed all .pxd files that are 1:1 of cugraph_c with a leading underscore, moved test fixtures to conftest.py. --- .../{algorithms.pxd => _algorithms.pxd} | 19 +-- .../_cugraph_c/{array.pxd => _array.pxd} | 4 +- .../{cugraph_api.pxd => _cugraph_api.pxd} | 0 .../_cugraph_c/{error.pxd => _error.pxd} | 0 .../_cugraph_c/{graph.pxd => _graph.pxd} | 6 +- .../_cugraph_c/graph_properties.pxd | 2 +- .../_cugraph_c/graph_properties.pyx | 21 +-- .../_cugraph_c/{sg_graph.pxd => graphs.pxd} | 10 +- .../_cugraph_c/{sg_graph.pyx => graphs.pyx} | 12 +- .../pylibcugraph/_cugraph_c/pagerank.pyx | 62 ++++++++ .../_cugraph_c/resource_handle.pxd | 2 +- .../_cugraph_c/resource_handle.pyx | 2 +- .../pylibcugraph/_cugraph_c/utils.pxd | 2 +- .../pylibcugraph/experimental/__init__.py | 2 +- .../pylibcugraph/tests/conftest.py | 136 ++++++++++++++++-- .../tests/test_connected_components.py | 26 ++-- .../{test_sg_graph.py => test_graph_sg.py} | 84 +++-------- .../pylibcugraph/tests/test_pagerank.py | 31 ++++ 18 files changed, 297 insertions(+), 124 deletions(-) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{algorithms.pxd => _algorithms.pxd} (85%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{array.pxd => _array.pxd} (96%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{cugraph_api.pxd => _cugraph_api.pxd} (100%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{error.pxd => _error.pxd} (100%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{graph.pxd => _graph.pxd} (94%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{sg_graph.pxd => graphs.pxd} (72%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{sg_graph.pyx => graphs.pyx} (95%) create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx rename python/pylibcugraph/pylibcugraph/tests/{test_sg_graph.py => test_graph_sg.py} (57%) create mode 100644 python/pylibcugraph/pylibcugraph/tests/test_pagerank.py diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd similarity index 85% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd index 2a7c4524c05..15c48ccc673 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd @@ -11,17 +11,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c.cugraph_api cimport ( +from pylibcugraph._cugraph_c._cugraph_api cimport ( bool_t, cugraph_resource_handle_t, ) -from pylibcugraph._cugraph_c.error cimport ( +from pylibcugraph._cugraph_c._error cimport ( cugraph_error_code_t, cugraph_error_t, ) -from pylibcugraph._cugraph_c.array cimport ( +from pylibcugraph._cugraph_c._array cimport ( cugraph_type_erased_device_array_t, ) +from pylibcugraph._cugraph_c._graph cimport ( + cugraph_graph_t, +) cdef extern from "cugraph_c/algorithms.h": @@ -35,22 +38,22 @@ cdef extern from "cugraph_c/algorithms.h": # ctypedef struct cugraph_extract_paths_result_t: # pass - cdef cugraph_type_erased_device_array_t* + cdef cugraph_type_erased_device_array_t* \ cugraph_pagerank_result_get_vertices( cugraph_pagerank_result_t* result ) - cdef cugraph_type_erased_device_array_t* + cdef cugraph_type_erased_device_array_t* \ cugraph_pagerank_result_get_pageranks( cugraph_pagerank_result_t* result ) - cdef void + cdef void \ cugraph_pagerank_result_free( cugraph_pagerank_result_t* result ) - cdef cugraph_error_code_t + cdef cugraph_error_code_t \ cugraph_pagerank( const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, @@ -64,7 +67,7 @@ cdef extern from "cugraph_c/algorithms.h": cugraph_error_t** error ) - cdef cugraph_error_code_t + cdef cugraph_error_code_t \ cugraph_personalized_pagerank( const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd similarity index 96% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd index 6239f9f174c..0366a94655e 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd @@ -11,11 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c.error cimport ( +from pylibcugraph._cugraph_c._error cimport ( cugraph_error_code_t, cugraph_error_t, ) -from pylibcugraph._cugraph_c.cugraph_api cimport ( +from pylibcugraph._cugraph_c._cugraph_api cimport ( cugraph_resource_handle_t, data_type_id_t, byte_t, diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd similarity index 100% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/error.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd similarity index 100% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/error.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd similarity index 94% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd index 97af3ba93d2..3dc3bfdc0d9 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd @@ -11,15 +11,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c.cugraph_api cimport ( +from pylibcugraph._cugraph_c._cugraph_api cimport ( bool_t, cugraph_resource_handle_t, ) -from pylibcugraph._cugraph_c.error cimport ( +from pylibcugraph._cugraph_c._error cimport ( cugraph_error_code_t, cugraph_error_t, ) -from pylibcugraph._cugraph_c.array cimport ( +from pylibcugraph._cugraph_c._array cimport ( cugraph_type_erased_device_array_t, cugraph_type_erased_host_array_t, ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pxd index 37511503360..d2cedcb3316 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pxd @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c.graph cimport ( +from pylibcugraph._cugraph_c._graph cimport ( cugraph_graph_properties_t, ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pyx index e0c6a6af77d..d178e4c7318 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pyx @@ -11,17 +11,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c.cugraph_api cimport ( - bool_t, -) - cdef class EXPERIMENTAL__GraphProperties: """ + Class wrapper around C cugraph_graph_properties_t struct """ - def __cinit__(self): - self.c_graph_properties.is_symmetric = bool_t.FALSE - self.c_graph_properties.is_multigraph = bool_t.FALSE + def __cinit__(self, is_symmetric=False, is_multigraph=False): + self.c_graph_properties.is_symmetric = is_symmetric + self.c_graph_properties.is_multigraph = is_multigraph @property def is_symmetric(self): @@ -29,10 +26,7 @@ cdef class EXPERIMENTAL__GraphProperties: @is_symmetric.setter def is_symmetric(self, value): - if not(isinstance(value, (int, bool))): - raise TypeError(f"expected int or bool, got {type(value)}") - - self.c_graph_properties.is_symmetric = int(value) + self.c_graph_properties.is_symmetric = value @property def is_multigraph(self): @@ -40,7 +34,4 @@ cdef class EXPERIMENTAL__GraphProperties: @is_multigraph.setter def is_multigraph(self, value): - if not(isinstance(value, (int, bool))): - raise TypeError(f"expected int or bool, got {type(value)}") - - self.c_graph_properties.is_multigraph = int(value) + self.c_graph_properties.is_multigraph = value diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pxd similarity index 72% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pxd index 92b6c74ee94..8491446bcb3 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pxd @@ -11,10 +11,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c.graph cimport ( +from pylibcugraph._cugraph_c._graph cimport ( cugraph_graph_t, ) -cdef class EXPERIMENTAL__SGGraph: +cdef class EXPERIMENTAL__Graph: + pass + +cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): cdef cugraph_graph_t* c_sg_graph_ptr + +# cdef class EXPERIMENTAL__MGGraph(EXPERIMENTAL__Graph): +# cdef cugraph_graph_t* c_mg_graph_ptr diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx similarity index 95% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx rename to python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx index 57be62bb47e..7530c00a9b4 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/sg_graph.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx @@ -11,23 +11,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -from libc.stdint cimport uintptr_t - -from pylibcugraph._cugraph_c.cugraph_api cimport ( +from pylibcugraph._cugraph_c._cugraph_api cimport ( bool_t, cugraph_resource_handle_t, data_type_id_t, ) -from pylibcugraph._cugraph_c.error cimport ( +from pylibcugraph._cugraph_c._error cimport ( cugraph_error_code_t, cugraph_error_t, ) -from pylibcugraph._cugraph_c.array cimport ( +from pylibcugraph._cugraph_c._array cimport ( cugraph_type_erased_device_array_t, cugraph_type_erased_device_array_create, cugraph_type_erased_device_array_free, ) -from pylibcugraph._cugraph_c.graph cimport ( +from pylibcugraph._cugraph_c._graph cimport ( cugraph_graph_t, cugraph_sg_graph_create, cugraph_graph_properties_t, @@ -44,7 +42,7 @@ from pylibcugraph._cugraph_c.utils cimport ( ) -cdef class EXPERIMENTAL__SGGraph: +cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): """ RAII-stye Graph class for use with single-GPU APIs that manages the individual create/free calls and the corresponding cugraph_graph_t pointer. diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx new file mode 100644 index 00000000000..dd29652638f --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx @@ -0,0 +1,62 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph._cugraph_c._cugraph_api cimport ( + bool_t, + #data_type_id_t, +) +from pylibcugraph._cugraph_c._error cimport ( + cugraph_error_code_t, + cugraph_error_t, +) +from pylibcugraph._cugraph_c._array cimport ( + cugraph_type_erased_device_array_t, + cugraph_type_erased_device_array_create, + cugraph_type_erased_device_array_free, +) +from pylibcugraph._cugraph_c._graph cimport ( + cugraph_graph_t, + cugraph_sg_graph_create, + cugraph_graph_properties_t, + cugraph_sg_graph_free, +) +from pylibcugraph._cugraph_c._algorithms cimport ( + cugraph_pagerank_result_t, +) + +from pylibcugraph._cugraph_c.resource_handle cimport ( + EXPERIMENTAL__ResourceHandle, +) +from pylibcugraph._cugraph_c.graphs cimport ( + EXPERIMENTAL__Graph, +) +from pylibcugraph._cugraph_c.utils cimport ( + assert_success, +) + + +cdef cugraph_error_code_t \ + pagerank(EXPERIMENTAL__ResourceHandle resource_handle, + EXPERIMENTAL__Graph graph, + cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, + double alpha, + double epsilon, + size_t max_iterations, + bool_t has_initial_guess, + bool_t do_expensive_check, + cugraph_pagerank_result_t** result, + cugraph_error_t** error + ): + """ + """ + pass diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pxd index afe6ba18a27..5bf16ae43c9 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pxd @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c.cugraph_api cimport ( +from pylibcugraph._cugraph_c._cugraph_api cimport ( cugraph_resource_handle_t, ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx index 30fc62ffd6d..1a008166793 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c.cugraph_api cimport ( +from pylibcugraph._cugraph_c._cugraph_api cimport ( cugraph_create_resource_handle, cugraph_free_resource_handle, ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pxd index 1836196124b..5965cc66364 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pxd @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c.error cimport ( +from pylibcugraph._cugraph_c._error cimport ( cugraph_error_code_t, cugraph_error_t, ) diff --git a/python/pylibcugraph/pylibcugraph/experimental/__init__.py b/python/pylibcugraph/pylibcugraph/experimental/__init__.py index 72d21e92829..391017bfd22 100644 --- a/python/pylibcugraph/pylibcugraph/experimental/__init__.py +++ b/python/pylibcugraph/pylibcugraph/experimental/__init__.py @@ -13,7 +13,7 @@ from pylibcugraph.utilities.api_tools import experimental_warning_wrapper -from pylibcugraph._cugraph_c.sg_graph import EXPERIMENTAL__SGGraph +from pylibcugraph._cugraph_c.graphs import EXPERIMENTAL__SGGraph SGGraph = experimental_warning_wrapper(EXPERIMENTAL__SGGraph) from pylibcugraph._cugraph_c.resource_handle import EXPERIMENTAL__ResourceHandle diff --git a/python/pylibcugraph/pylibcugraph/tests/conftest.py b/python/pylibcugraph/pylibcugraph/tests/conftest.py index dae3c7fe2e1..62f0ca28959 100644 --- a/python/pylibcugraph/pylibcugraph/tests/conftest.py +++ b/python/pylibcugraph/pylibcugraph/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. +# Copyright (c) 2021-2022, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,15 +11,135 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pandas as pd +import cupy as cp +import numpy as np + import pytest +from . import utils + + +# ============================================================================= +# Fixture parameters +# ============================================================================= +class InlineGraphData: + @property + def name(self): + return self.__class__.__name__ + + @property + def is_valid(self): + return not(self.name.startswith("Invalid")) + + +class InvalidNumWeights_1(InlineGraphData): + srcs = cp.asarray([0, 1, 2], dtype=np.int32) + dsts = cp.asarray([1, 2, 3], dtype=np.int32) + weights = cp.asarray([0, 0, 0, 0], dtype=np.int32) + + +class InvalidNumVerts_1(InlineGraphData): + srcs = cp.asarray([1, 2], dtype=np.int32) + dsts = cp.asarray([1, 2, 3], dtype=np.int32) + weights = cp.asarray([0, 0, 0], dtype=np.int32) + + +class Simple_1(InlineGraphData): + srcs = cp.asarray([0, 1, 2], dtype=np.int32) + dsts = cp.asarray([1, 2, 3], dtype=np.int32) + weights = cp.asarray([0, 0, 0], dtype=np.int32) + + +valid_datasets = [utils.RAPIDS_DATASET_ROOT_DIR_PATH/"karate.csv", + utils.RAPIDS_DATASET_ROOT_DIR_PATH/"dolphins.csv", + Simple_1(), + ] +all_datasets = valid_datasets + \ + [InvalidNumWeights_1(), + InvalidNumVerts_1(), + ] + +# ============================================================================= +# Helper functions +# ============================================================================= +def get_graph_data_for_dataset(ds): + """ + Given an object representing either a path to a dataset on disk, or an + object containing raw data, return a series of arrays that can be used to + construct a graph object. The final value is a bool used to indicate if the + data is valid or not (invalid to test error handling). + """ + if isinstance(ds, InlineGraphData): + device_srcs = ds.srcs + device_dsts = ds.dsts + device_weights = ds.weights + is_valid = ds.is_valid + else: + pdf = pd.read_csv(ds, + delimiter=" ", header=None, + names=["0", "1", "weight"], + dtype={"0": "int32", "1": "int32", + "weight": "float32"}, + ) + device_srcs = cp.asarray(pdf["0"].to_numpy(), dtype=np.int32) + device_dsts = cp.asarray(pdf["1"].to_numpy(), dtype=np.int32) + device_weights = cp.asarray(pdf["weight"].to_numpy(), dtype=np.float32) + # Assume all datasets on disk are valid + is_valid = True + + return (device_srcs, device_dsts, device_weights, is_valid) + +# ============================================================================= +# Pytest fixtures +# ============================================================================= +@pytest.fixture(scope="package", + params=[pytest.param(ds, id=ds.name) for ds in all_datasets]) +def graph_data(request): + """ + Return a series of cupy arrays that can be used to construct Graph + objects. The parameterization includes invalid arrays which can be used to + error handling, so the final value returned indicated if the arrays are + valid or not. + """ + return get_graph_data_for_dataset(request.param) + + +@pytest.fixture(scope="package", + params=[pytest.param(ds, id=ds.name) for ds in valid_datasets]) +def valid_graph_data(request): + """ + Return a series of cupy arrays that can be used to construct Graph objects, + all of which are valid. + """ + return get_graph_data_for_dataset(request.param) -@pytest.fixture -def package_under_test(): + +@pytest.fixture(scope="package") +def sg_graph(valid_graph_data, request): """ - Create a fixture to import the package under test. This is useful since - bugs that prevent the package under test from being imported will not - prevent pytest from collecting, listing, running, etc. the tests. + Returns a SGGraph object constructed from parameterized values returned by + the valid_graph_data fixture. """ - import pylibcugraph - return pylibcugraph + from pylibcugraph.experimental import (SGGraph, + ResourceHandle, + GraphProperties, + ) + + (device_srcs, device_dsts, device_weights, is_valid) = valid_graph_data + + if is_valid is False: + pytest.exit("got invalid graph data - expecting only valid data") + + graph_props = GraphProperties(is_symmetric=False, is_multigraph=False) + resource_handle = ResourceHandle() + + g = SGGraph(resource_handle, + graph_props, + device_srcs, + device_dsts, + device_weights, + store_transposed=False, + renumber=False, + expensive_check=False) + return g diff --git a/python/pylibcugraph/pylibcugraph/tests/test_connected_components.py b/python/pylibcugraph/pylibcugraph/tests/test_connected_components.py index e4316bbdc47..71b4314fc66 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_connected_components.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_connected_components.py @@ -21,6 +21,10 @@ from . import utils + +# ============================================================================= +# Test data +# ============================================================================= _test_data = { "graph1": # asymmetric { @@ -96,6 +100,9 @@ } +# ============================================================================= +# Pytest fixtures and helpers +# ============================================================================= @pytest.fixture(scope="module", params=[pytest.param(value, id=key) for (key, value) in _test_data.items()]) @@ -172,8 +179,9 @@ def _check_labels(vertex_ordered_labels, expected_vertex_comps): assert actual_vertex_comps == sorted(expected_vertex_comps) -############################################################################### +# ============================================================================= # Tests +# ============================================================================= def test_import(): """ Ensure pylibcugraph is importable. @@ -182,11 +190,11 @@ def test_import(): import pylibcugraph # noqa: F401 -def test_scc(package_under_test, input_and_expected_output): +def test_scc(input_and_expected_output): """ Tests strongly_connected_components() """ - pylibcugraph = package_under_test + import pylibcugraph ((cupy_offsets, cupy_indices, cupy_labels_to_populate, num_verts, num_edges), expected_output_dict) = input_and_expected_output @@ -204,11 +212,11 @@ def test_scc(package_under_test, input_and_expected_output): expected_output_dict["scc_comp_vertices"]) -def test_wcc(package_under_test, input_and_expected_output): +def test_wcc(input_and_expected_output): """ Tests weakly_connected_components() """ - pylibcugraph = package_under_test + import pylibcugraph ((cupy_offsets, cupy_indices, cupy_labels_to_populate, num_verts, num_edges), expected_output_dict) = input_and_expected_output @@ -228,12 +236,12 @@ def test_wcc(package_under_test, input_and_expected_output): @pytest.mark.parametrize("api_name", ["strongly_connected_components", "weakly_connected_components"]) -def test_non_CAI_input(package_under_test, api_name): +def test_non_CAI_input(api_name): """ Ensures that the *_connected_components() APIs only accepts instances of objects that have a __cuda_array_interface__ """ - pylibcugraph = package_under_test + import pylibcugraph cupy_array = cp.ndarray(range(8)) python_list = list(range(8)) api = getattr(pylibcugraph, api_name) @@ -270,10 +278,11 @@ def test_non_CAI_input(package_under_test, api_name): @pytest.mark.parametrize("api_name", ["strongly_connected_components", "weakly_connected_components"]) -def test_bad_dtypes(package_under_test, api_name): +def test_bad_dtypes(api_name): """ Ensures that only supported dtypes are accepted. """ + import pylibcugraph graph = [ [0, 1, 1, 0, 0], [0, 0, 1, 0, 0], @@ -285,7 +294,6 @@ def test_bad_dtypes(package_under_test, api_name): num_verts = scipy_csr.get_shape()[0] num_edges = scipy_csr.nnz - pylibcugraph = package_under_test api = getattr(pylibcugraph, api_name) cp_offsets = cp.asarray(scipy_csr.indptr) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py b/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py similarity index 57% rename from python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py rename to python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py index f334d74f842..b455579397b 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_sg_graph.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py @@ -23,67 +23,12 @@ # ============================================================================= # Pytest fixtures # ============================================================================= -class InlineGraphData: - @property - def name(self): - return self.__class__.__name__ - - @property - def is_valid(self): - return not(self.name.startswith("Invalid")) - -class InvalidNumWeights_1(InlineGraphData): # noqa: E302 - srcs = cp.asarray([0, 1, 2], dtype=np.int32) - dsts = cp.asarray([1, 2, 3], dtype=np.int32) - weights = cp.asarray([0, 0, 0, 0], dtype=np.int32) - -class InvalidNumVerts_1(InlineGraphData): # noqa: E302 - srcs = cp.asarray([1, 2], dtype=np.int32) - dsts = cp.asarray([1, 2, 3], dtype=np.int32) - weights = cp.asarray([0, 0, 0], dtype=np.int32) - -class Simple_1(InlineGraphData): # noqa: E302 - srcs = cp.asarray([0, 1, 2], dtype=np.int32) - dsts = cp.asarray([1, 2, 3], dtype=np.int32) - weights = cp.asarray([0, 0, 0], dtype=np.int32) - - -datasets = [utils.RAPIDS_DATASET_ROOT_DIR_PATH/"karate.csv", - utils.RAPIDS_DATASET_ROOT_DIR_PATH/"dolphins.csv", - InvalidNumWeights_1(), - InvalidNumVerts_1(), - Simple_1(), - ] - - -@pytest.fixture(scope="module", - params=[pytest.param(ds, id=ds.name) for ds in datasets]) -def graph_data(request): - ds = request.param - - if isinstance(ds, InlineGraphData): - device_srcs = ds.srcs - device_dsts = ds.dsts - device_weights = ds.weights - is_valid = ds.is_valid - else: - pdf = pd.read_csv(ds, - delimiter=" ", header=None, - names=["0", "1", "weight"], - dtype={"0": "int32", "1": "int32", - "weight": "float32"}, - ) - device_srcs = cp.asarray(pdf["0"].to_numpy(), dtype=np.int32) - device_dsts = cp.asarray(pdf["1"].to_numpy(), dtype=np.int32) - device_weights = cp.asarray(pdf["weight"].to_numpy(), dtype=np.float32) - # Assume all datasets on disk are valid - is_valid = True - - return (device_srcs, device_dsts, device_weights, is_valid) - - -############################################################################### +# fixtures used in this test module are defined in conftest.py + + +# ============================================================================= # Tests +# ============================================================================= def test_graph_properties(): from pylibcugraph.experimental import GraphProperties @@ -105,6 +50,16 @@ def test_graph_properties(): with pytest.raises(TypeError): gp.is_multigraph = "foo" + gp = GraphProperties(is_symmetric=True, is_multigraph=True) + assert gp.is_symmetric is True + assert gp.is_multigraph is True + + with pytest.raises(TypeError): + gp = GraphProperties(is_symmetric="foo", is_multigraph=False) + + with pytest.raises(TypeError): + gp = GraphProperties(is_multigraph=[]) + def test_resource_handle(): from pylibcugraph.experimental import ResourceHandle @@ -115,8 +70,7 @@ def test_resource_handle(): del rh -def test_sg_graph_ctor(graph_data): - +def test_sg_graph(graph_data): from pylibcugraph.experimental import (SGGraph, ResourceHandle, GraphProperties, @@ -138,8 +92,9 @@ def test_sg_graph_ctor(graph_data): store_transposed=False, renumber=False, expensive_check=False) + # explicitly run __dealloc__() + del g - print(g) else: with pytest.raises(RuntimeError): g = SGGraph(resource_handle, @@ -150,5 +105,4 @@ def test_sg_graph_ctor(graph_data): store_transposed=False, renumber=False, expensive_check=False) - - print(g) + del g diff --git a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py new file mode 100644 index 00000000000..cdab705a47c --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import cupy as cp +import numpy as np + + +# ============================================================================= +# Pytest fixtures +# ============================================================================= +# fixtures used in this test module are defined in conftest.py + + +# ============================================================================= +# Tests +# ============================================================================= +def test_pagerank(sg_graph): + g = sg_graph + + print(g) From 4acd8aa8c8074ce759a6c0cb71c009de12a4d020 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Fri, 21 Jan 2022 10:24:49 -0600 Subject: [PATCH 13/43] Test refactoring to better handle fixtures that provide inputs and associated expected outputs, more updates to pagerank function and test. --- .../pylibcugraph/_cugraph_c/graphs.pxd | 6 +-- .../pylibcugraph/_cugraph_c/graphs.pyx | 24 +++++----- .../pylibcugraph/_cugraph_c/pagerank.pyx | 46 ++++++++++++------- .../pylibcugraph/experimental/__init__.py | 3 ++ .../pylibcugraph/tests/conftest.py | 30 ++++++++---- .../pylibcugraph/tests/test_graph_sg.py | 2 +- .../pylibcugraph/tests/test_pagerank.py | 34 ++++++++++++-- 7 files changed, 99 insertions(+), 46 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pxd index 8491446bcb3..dfb5be5426b 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pxd @@ -17,10 +17,10 @@ from pylibcugraph._cugraph_c._graph cimport ( cdef class EXPERIMENTAL__Graph: - pass + cdef cugraph_graph_t* c_graph_ptr cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): - cdef cugraph_graph_t* c_sg_graph_ptr + pass # cdef class EXPERIMENTAL__MGGraph(EXPERIMENTAL__Graph): -# cdef cugraph_graph_t* c_mg_graph_ptr +# pass diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx index 7530c00a9b4..a7e528bf559 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx @@ -78,52 +78,52 @@ cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): "__cuda_array_interface__ attr") cdef cugraph_error_t* error_ptr - cdef cugraph_error_code_t err_code + cdef cugraph_error_code_t error_code cdef cugraph_type_erased_device_array_t* srcs_ptr cdef cugraph_type_erased_device_array_t* dsts_ptr cdef cugraph_type_erased_device_array_t* weights_ptr # FIXME: set dtype properly - err_code = cugraph_type_erased_device_array_create( + error_code = cugraph_type_erased_device_array_create( resource_handle.c_resource_handle_ptr, data_type_id_t.INT32, len(src_array), &srcs_ptr, &error_ptr) - assert_success(err_code, error_ptr, + assert_success(error_code, error_ptr, "cugraph_type_erased_device_array_create()") # FIXME: add call to to device-device copy of __cuda_array_interface__ # values to cugraph_type_erased_device_array # FIXME: set dtype properly - err_code = cugraph_type_erased_device_array_create( + error_code = cugraph_type_erased_device_array_create( resource_handle.c_resource_handle_ptr, data_type_id_t.INT32, len(dst_array), &dsts_ptr, &error_ptr) - assert_success(err_code, error_ptr, + assert_success(error_code, error_ptr, "cugraph_type_erased_device_array_create()") # FIXME: add call to to device-device copy of __cuda_array_interface__ # values to cugraph_type_erased_device_array # FIXME: set dtype properly - err_code = cugraph_type_erased_device_array_create( + error_code = cugraph_type_erased_device_array_create( resource_handle.c_resource_handle_ptr, data_type_id_t.FLOAT32, len(weight_array), &weights_ptr, &error_ptr) - assert_success(err_code, error_ptr, + assert_success(error_code, error_ptr, "cugraph_type_erased_device_array_create()") # FIXME: add call to to device-device copy of __cuda_array_interface__ # values to cugraph_type_erased_device_array - err_code = cugraph_sg_graph_create( + error_code = cugraph_sg_graph_create( resource_handle.c_resource_handle_ptr, &(graph_properties.c_graph_properties), srcs_ptr, @@ -132,12 +132,12 @@ cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): int(store_transposed), int(renumber), int(expensive_check), - &(self.c_sg_graph_ptr), + &(self.c_graph_ptr), &error_ptr) - assert_success(err_code, error_ptr, + assert_success(error_code, error_ptr, "cugraph_sg_graph_create()") def __dealloc__(self): - if self.c_sg_graph_ptr is not NULL: - cugraph_sg_graph_free(self.c_sg_graph_ptr) + if self.c_graph_ptr is not NULL: + cugraph_sg_graph_free(self.c_graph_ptr) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx index dd29652638f..c79181f2402 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx @@ -14,6 +14,7 @@ from pylibcugraph._cugraph_c._cugraph_api cimport ( bool_t, #data_type_id_t, + cugraph_resource_handle_t, ) from pylibcugraph._cugraph_c._error cimport ( cugraph_error_code_t, @@ -26,12 +27,10 @@ from pylibcugraph._cugraph_c._array cimport ( ) from pylibcugraph._cugraph_c._graph cimport ( cugraph_graph_t, - cugraph_sg_graph_create, - cugraph_graph_properties_t, - cugraph_sg_graph_free, ) from pylibcugraph._cugraph_c._algorithms cimport ( cugraph_pagerank_result_t, + cugraph_pagerank, ) from pylibcugraph._cugraph_c.resource_handle cimport ( @@ -45,18 +44,33 @@ from pylibcugraph._cugraph_c.utils cimport ( ) -cdef cugraph_error_code_t \ - pagerank(EXPERIMENTAL__ResourceHandle resource_handle, - EXPERIMENTAL__Graph graph, - cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, - double alpha, - double epsilon, - size_t max_iterations, - bool_t has_initial_guess, - bool_t do_expensive_check, - cugraph_pagerank_result_t** result, - cugraph_error_t** error - ): + +def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, + EXPERIMENTAL__Graph graph, + precomputed_vertex_out_weight_sums, + double alpha, + double epsilon, + size_t max_iterations, + bool_t has_initial_guess, + bool_t do_expensive_check): """ """ - pass + cdef cugraph_resource_handle_t* c_resource_handle_ptr = resource_handle.c_resource_handle_ptr + cdef cugraph_graph_t* c_graph_ptr = graph.c_graph_ptr + #cdef cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums_ptr = precomputed_vertex_out_weight_sums + cdef cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums_ptr = NULL + + cdef cugraph_pagerank_result_t* result_ptr + cdef cugraph_error_code_t error_code + cdef cugraph_error_t* error_ptr + + error_code = cugraph_pagerank(c_resource_handle_ptr, + c_graph_ptr, + precomputed_vertex_out_weight_sums_ptr, + alpha, + epsilon, + max_iterations, + has_initial_guess, + do_expensive_check, + &result_ptr, + &error_ptr) diff --git a/python/pylibcugraph/pylibcugraph/experimental/__init__.py b/python/pylibcugraph/pylibcugraph/experimental/__init__.py index 391017bfd22..d0c65823e89 100644 --- a/python/pylibcugraph/pylibcugraph/experimental/__init__.py +++ b/python/pylibcugraph/pylibcugraph/experimental/__init__.py @@ -21,3 +21,6 @@ from pylibcugraph._cugraph_c.graph_properties import EXPERIMENTAL__GraphProperties GraphProperties = experimental_warning_wrapper(EXPERIMENTAL__GraphProperties) + +from pylibcugraph._cugraph_c.pagerank import EXPERIMENTAL__pagerank +pagerank = experimental_warning_wrapper(EXPERIMENTAL__pagerank) diff --git a/python/pylibcugraph/pylibcugraph/tests/conftest.py b/python/pylibcugraph/pylibcugraph/tests/conftest.py index 62f0ca28959..d62fb4bc1dc 100644 --- a/python/pylibcugraph/pylibcugraph/tests/conftest.py +++ b/python/pylibcugraph/pylibcugraph/tests/conftest.py @@ -50,10 +50,19 @@ class Simple_1(InlineGraphData): dsts = cp.asarray([1, 2, 3], dtype=np.int32) weights = cp.asarray([0, 0, 0], dtype=np.int32) +class Simple_2(InlineGraphData): + srcs = cp.asarray([0, 1, 1, 2, 2, 2, 3, 4], dtype=np.int32) + dsts = cp.asarray([1, 3, 4, 0, 1, 3, 5, 5], dtype=np.int32) + weights = cp.asarray([0.1, 2.1, 1.1, 5.1, 3.1, 4.1, 7.2, 3.2], dtype=np.float32) + +# The objects in these lists must have a "name" attr, since fixtures will access +# that to pass to tests, which then may use the name to associate to expected +# test results. The name attr is also used for the pytest test ID valid_datasets = [utils.RAPIDS_DATASET_ROOT_DIR_PATH/"karate.csv", utils.RAPIDS_DATASET_ROOT_DIR_PATH/"dolphins.csv", Simple_1(), + Simple_2(), ] all_datasets = valid_datasets + \ [InvalidNumWeights_1(), @@ -63,7 +72,7 @@ class Simple_1(InlineGraphData): # ============================================================================= # Helper functions # ============================================================================= -def get_graph_data_for_dataset(ds): +def get_graph_data_for_dataset(ds, ds_name): """ Given an object representing either a path to a dataset on disk, or an object containing raw data, return a series of arrays that can be used to @@ -88,7 +97,7 @@ def get_graph_data_for_dataset(ds): # Assume all datasets on disk are valid is_valid = True - return (device_srcs, device_dsts, device_weights, is_valid) + return (device_srcs, device_dsts, device_weights, ds_name, is_valid) # ============================================================================= # Pytest fixtures @@ -102,7 +111,7 @@ def graph_data(request): error handling, so the final value returned indicated if the arrays are valid or not. """ - return get_graph_data_for_dataset(request.param) + return get_graph_data_for_dataset(request.param, request.param.name) @pytest.fixture(scope="package", @@ -112,21 +121,24 @@ def valid_graph_data(request): Return a series of cupy arrays that can be used to construct Graph objects, all of which are valid. """ - return get_graph_data_for_dataset(request.param) + return get_graph_data_for_dataset(request.param, request.param.name) @pytest.fixture(scope="package") -def sg_graph(valid_graph_data, request): +def sg_graph_objs(valid_graph_data, request): """ - Returns a SGGraph object constructed from parameterized values returned by - the valid_graph_data fixture. + Returns a tuple containing the SGGraph object constructed from + parameterized values returned by the valid_graph_data fixture, + the associated resource handle, and the name of the dataset + used to construct the graph. """ from pylibcugraph.experimental import (SGGraph, ResourceHandle, GraphProperties, ) - (device_srcs, device_dsts, device_weights, is_valid) = valid_graph_data + (device_srcs, device_dsts, device_weights, ds_name, is_valid) = \ + valid_graph_data if is_valid is False: pytest.exit("got invalid graph data - expecting only valid data") @@ -142,4 +154,4 @@ def sg_graph(valid_graph_data, request): store_transposed=False, renumber=False, expensive_check=False) - return g + return (g, resource_handle, ds_name) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py b/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py index b455579397b..9008da6b389 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py @@ -76,7 +76,7 @@ def test_sg_graph(graph_data): GraphProperties, ) - (device_srcs, device_dsts, device_weights, is_valid) = graph_data + (device_srcs, device_dsts, device_weights, ds_name, is_valid) = graph_data graph_props = GraphProperties() graph_props.is_symmetric = False diff --git a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py index cdab705a47c..e45ad75a5ee 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py @@ -19,13 +19,37 @@ # ============================================================================= # Pytest fixtures # ============================================================================= -# fixtures used in this test module are defined in conftest.py - +# sg_graph_and_resource_handle fixture is defined in conftest.py. That fixture +# returns a preconstructed graph and corresponding resource handle for +# different datasets. +@pytest.fixture +def input_and_expected_output(sg_graph_objs): + (g, resource_handle, ds_name) = sg_graph_objs + return (g, resource_handle) # ============================================================================= # Tests # ============================================================================= -def test_pagerank(sg_graph): - g = sg_graph +@pytest.mark.skip(reason="UNFINISHED") +def test_pagerank(input_and_expected_output): + from pylibcugraph.experimental import pagerank + + (g, resource_handle) = input_and_expected_output + + precomputed_vertex_out_weight_sums = None + alpha = 0.95 + epsilon = 0.0001 + max_iterations = 20 + has_initial_guess = False + do_expensive_check = False + + result = pagerank(resource_handle, + g, + precomputed_vertex_out_weight_sums, + alpha, + epsilon, + max_iterations, + has_initial_guess, + do_expensive_check) - print(g) + raise NotImplementedError From b340d7ae3e40f4d67544d0ecf7593d1b6450b0ef Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Fri, 21 Jan 2022 10:54:54 -0600 Subject: [PATCH 14/43] flake8 fixes. --- .../pylibcugraph/tests/conftest.py | 14 ++++++---- .../pylibcugraph/tests/test_graph_sg.py | 27 +++++++------------ .../pylibcugraph/tests/test_pagerank.py | 7 ++--- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/tests/conftest.py b/python/pylibcugraph/pylibcugraph/tests/conftest.py index d62fb4bc1dc..ce668f66fae 100644 --- a/python/pylibcugraph/pylibcugraph/tests/conftest.py +++ b/python/pylibcugraph/pylibcugraph/tests/conftest.py @@ -50,25 +50,28 @@ class Simple_1(InlineGraphData): dsts = cp.asarray([1, 2, 3], dtype=np.int32) weights = cp.asarray([0, 0, 0], dtype=np.int32) + class Simple_2(InlineGraphData): srcs = cp.asarray([0, 1, 1, 2, 2, 2, 3, 4], dtype=np.int32) dsts = cp.asarray([1, 3, 4, 0, 1, 3, 5, 5], dtype=np.int32) - weights = cp.asarray([0.1, 2.1, 1.1, 5.1, 3.1, 4.1, 7.2, 3.2], dtype=np.float32) + weights = cp.asarray([0.1, 2.1, 1.1, 5.1, 3.1, 4.1, 7.2, 3.2], + dtype=np.float32) -# The objects in these lists must have a "name" attr, since fixtures will access -# that to pass to tests, which then may use the name to associate to expected -# test results. The name attr is also used for the pytest test ID +# The objects in these lists must have a "name" attr, since fixtures will +# access that to pass to tests, which then may use the name to associate to +# expected test results. The name attr is also used for the pytest test ID. valid_datasets = [utils.RAPIDS_DATASET_ROOT_DIR_PATH/"karate.csv", utils.RAPIDS_DATASET_ROOT_DIR_PATH/"dolphins.csv", Simple_1(), Simple_2(), - ] + ] all_datasets = valid_datasets + \ [InvalidNumWeights_1(), InvalidNumVerts_1(), ] + # ============================================================================= # Helper functions # ============================================================================= @@ -99,6 +102,7 @@ def get_graph_data_for_dataset(ds, ds_name): return (device_srcs, device_dsts, device_weights, ds_name, is_valid) + # ============================================================================= # Pytest fixtures # ============================================================================= diff --git a/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py b/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py index 9008da6b389..ad0195e92e1 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py @@ -11,14 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pandas as pd -import cupy as cp -import numpy as np - import pytest -from . import utils - # ============================================================================= # Pytest fixtures @@ -84,7 +78,7 @@ def test_sg_graph(graph_data): resource_handle = ResourceHandle() if is_valid: - g = SGGraph(resource_handle, + g = SGGraph(resource_handle, # noqa:F841 graph_props, device_srcs, device_dsts, @@ -92,17 +86,16 @@ def test_sg_graph(graph_data): store_transposed=False, renumber=False, expensive_check=False) - # explicitly run __dealloc__() + # call SGGraph.__dealloc__() del g else: with pytest.raises(RuntimeError): - g = SGGraph(resource_handle, - graph_props, - device_srcs, - device_dsts, - device_weights, - store_transposed=False, - renumber=False, - expensive_check=False) - del g + SGGraph(resource_handle, + graph_props, + device_srcs, + device_dsts, + device_weights, + store_transposed=False, + renumber=False, + expensive_check=False) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py index e45ad75a5ee..41d3441401c 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py @@ -12,8 +12,8 @@ # limitations under the License. import pytest -import cupy as cp -import numpy as np +# import cupy as cp +# import numpy as np # ============================================================================= @@ -27,6 +27,7 @@ def input_and_expected_output(sg_graph_objs): (g, resource_handle, ds_name) = sg_graph_objs return (g, resource_handle) + # ============================================================================= # Tests # ============================================================================= @@ -51,5 +52,5 @@ def test_pagerank(input_and_expected_output): max_iterations, has_initial_guess, do_expensive_check) - + print(result) raise NotImplementedError From 7d5806bf8125005776513a56a2c22bad365c9acf Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Fri, 21 Jan 2022 11:02:53 -0600 Subject: [PATCH 15/43] Updated copyright year. --- .../pylibcugraph/tests/test_connected_components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_connected_components.py b/python/pylibcugraph/pylibcugraph/tests/test_connected_components.py index 71b4314fc66..7729b979bd1 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_connected_components.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_connected_components.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. +# Copyright (c) 2021-2022, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at From 1c2fc0286b48792a4e2bba86f64833dd2dd0de48 Mon Sep 17 00:00:00 2001 From: Chuck Hastings Date: Fri, 21 Jan 2022 16:41:26 -0500 Subject: [PATCH 16/43] add view semantic for arrays. Add copy between device arrays --- cpp/include/cugraph_c/algorithms.h | 26 ++-- cpp/include/cugraph_c/array.h | 164 ++++++++++++++++++++----- cpp/include/cugraph_c/graph.h | 16 +-- cpp/src/c_api/array.cpp | 173 ++++++++++++++++++++++----- cpp/src/c_api/array.hpp | 53 ++++++-- cpp/src/c_api/bfs.cpp | 20 ++-- cpp/src/c_api/extract_paths.cpp | 26 ++-- cpp/src/c_api/graph_mg.cpp | 10 +- cpp/src/c_api/graph_sg.cpp | 25 ++-- cpp/src/c_api/pagerank.cpp | 36 +++--- cpp/tests/c_api/bfs_test.c | 17 +-- cpp/tests/c_api/create_graph_test.c | 30 +++-- cpp/tests/c_api/extract_paths_test.c | 59 ++++++--- cpp/tests/c_api/pagerank_test.c | 11 +- cpp/tests/c_api/test_utils.c | 34 ++++-- 15 files changed, 499 insertions(+), 201 deletions(-) diff --git a/cpp/include/cugraph_c/algorithms.h b/cpp/include/cugraph_c/algorithms.h index 91ff43e8449..11923647cb5 100644 --- a/cpp/include/cugraph_c/algorithms.h +++ b/cpp/include/cugraph_c/algorithms.h @@ -37,7 +37,7 @@ typedef struct { * @param [in] result The result from pagerank * @return type erased array of vertex ids */ -cugraph_type_erased_device_array_t* cugraph_pagerank_result_get_vertices( +cugraph_type_erased_device_array_view_t* cugraph_pagerank_result_get_vertices( cugraph_pagerank_result_t* result); /** @@ -46,7 +46,7 @@ cugraph_type_erased_device_array_t* cugraph_pagerank_result_get_vertices( * @param [in] result The result from pagerank * @return type erased array of pagerank values */ -cugraph_type_erased_device_array_t* cugraph_pagerank_result_get_pageranks( +cugraph_type_erased_device_array_view_t* cugraph_pagerank_result_get_pageranks( cugraph_pagerank_result_t* result); /** @@ -83,7 +83,7 @@ void cugraph_pagerank_result_free(cugraph_pagerank_result_t* result); cugraph_error_code_t cugraph_pagerank( const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, - const cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, + const cugraph_type_erased_device_array_view_t* precomputed_vertex_out_weight_sums, double alpha, double epsilon, size_t max_iterations, @@ -125,10 +125,10 @@ cugraph_error_code_t cugraph_pagerank( cugraph_error_code_t cugraph_personalized_pagerank( const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, - const cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, + const cugraph_type_erased_device_array_view_t* precomputed_vertex_out_weight_sums, // FIXME: Make this const, copy it if I need to temporarily modify internally - cugraph_type_erased_device_array_t* personalization_vertices, - const cugraph_type_erased_device_array_t* personalization_values, + cugraph_type_erased_device_array_view_t* personalization_vertices, + const cugraph_type_erased_device_array_view_t* personalization_values, double alpha, double epsilon, size_t max_iterations, @@ -153,7 +153,7 @@ typedef struct { * @param [in] result The result from bfs or sssp * @return type erased array of vertex ids */ -cugraph_type_erased_device_array_t* cugraph_paths_result_get_vertices( +cugraph_type_erased_device_array_view_t* cugraph_paths_result_get_vertices( cugraph_paths_result_t* result); /** @@ -162,7 +162,7 @@ cugraph_type_erased_device_array_t* cugraph_paths_result_get_vertices( * @param [in] result The result from bfs or sssp * @return type erased array of distances */ -cugraph_type_erased_device_array_t* cugraph_paths_result_get_distances( +cugraph_type_erased_device_array_view_t* cugraph_paths_result_get_distances( cugraph_paths_result_t* result); /** @@ -173,7 +173,7 @@ cugraph_type_erased_device_array_t* cugraph_paths_result_get_distances( * compute_predecessors was FALSE in the call to bfs or sssp that * produced this result. */ -cugraph_type_erased_device_array_t* cugraph_paths_result_get_predecessors( +cugraph_type_erased_device_array_view_t* cugraph_paths_result_get_predecessors( cugraph_paths_result_t* result); /** @@ -211,7 +211,7 @@ cugraph_error_code_t cugraph_bfs( const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, // FIXME: Make this const, copy it if I need to temporarily modify internally - cugraph_type_erased_device_array_t* sources, + cugraph_type_erased_device_array_view_t* sources, bool_t direction_optimizing, size_t depth_limit, bool_t do_expensive_check, @@ -247,9 +247,9 @@ typedef struct { */ cugraph_error_code_t cugraph_extract_paths(const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, - const cugraph_type_erased_device_array_t* sources, + const cugraph_type_erased_device_array_view_t* sources, const cugraph_paths_result_t* paths_result, - const cugraph_type_erased_device_array_t* destinations, + const cugraph_type_erased_device_array_view_t* destinations, cugraph_extract_paths_result_t** result, cugraph_error_t** error); @@ -267,7 +267,7 @@ size_t cugraph_extract_paths_result_get_max_path_length(cugraph_extract_paths_re * @param [in] result The result from extract_paths * @return type erased array pointing to the matrix in device memory */ -cugraph_type_erased_device_array_t* cugraph_extract_paths_result_get_paths( +cugraph_type_erased_device_array_view_t* cugraph_extract_paths_result_get_paths( cugraph_extract_paths_result_t* result); /** diff --git a/cpp/include/cugraph_c/array.h b/cpp/include/cugraph_c/array.h index 50b085fbd3b..1f24889800a 100644 --- a/cpp/include/cugraph_c/array.h +++ b/cpp/include/cugraph_c/array.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,16 +26,24 @@ typedef struct { int align_; } cugraph_type_erased_device_array_t; +typedef struct { + int align_; +} cugraph_type_erased_device_array_view_t; + typedef struct { int align_; } cugraph_type_erased_host_array_t; +typedef struct { + int align_; +} cugraph_type_erased_host_array_view_t; + /** * @brief Create a type erased device array * * @param [in] handle Handle for accessing resources - * @param [in] dtype The type of array to create * @param [in] n_elems The number of elements in the array + * @param [in] dtype The type of array to create * @param [out] array Pointer to the location to store the pointer to the device array * @param [out] error Pointer to an error object storing details of any error. Will * be populated if error code is not CUGRAPH_SUCCESS @@ -43,8 +51,8 @@ typedef struct { */ cugraph_error_code_t cugraph_type_erased_device_array_create( const cugraph_resource_handle_t* handle, - data_type_id_t dtype, size_t n_elems, + data_type_id_t dtype, cugraph_type_erased_device_array_t** array, cugraph_error_t** error); @@ -55,44 +63,87 @@ cugraph_error_code_t cugraph_type_erased_device_array_create( */ void cugraph_type_erased_device_array_free(cugraph_type_erased_device_array_t* p); +#if 0 /** - * @brief Get the size of a type erased device array + * @brief Release the raw pointer of the type erased device array + * + * The caller is now responsible for freeing the device pointer * * @param [in] p Pointer to the type erased device array + * @return Pointer (device memory) for the data in the array + */ +void* cugraph_type_erased_device_array_release(cugraph_type_erased_device_array_t* p); +#endif + +/** + * @brief Create a type erased device array view from + * a type erased device array + * + * @param [in] array Pointer to the type erased device array + * @return Pointer to the view of the host array + */ +cugraph_type_erased_device_array_view_t* cugraph_type_erased_device_array_view( + cugraph_type_erased_device_array_t* array); + +/** + * @brief Create a type erased device array view from + * a raw device pointer. + * + * @param [in] pointer Raw device pointer + * @param [in] n_elems The number of elements in the array + * @param [in] dtype The type of array to create + * @return Pointer to the view of the host array + */ +cugraph_type_erased_device_array_view_t* cugraph_type_erased_device_array_view_create( + void* pointer, size_t n_elems, data_type_id_t dtype); + +/** + * @brief Destroy a type erased device array view + * + * @param [in] p Pointer to the type erased device array view + */ +void cugraph_type_erased_device_array_view_free(cugraph_type_erased_device_array_view_t* p); + +/** + * @brief Get the size of a type erased device array view + * + * @param [in] p Pointer to the type erased device array view * @return The number of elements in the array */ -size_t cugraph_type_erased_device_array_size(const cugraph_type_erased_device_array_t* p); +size_t cugraph_type_erased_device_array_view_size(const cugraph_type_erased_device_array_view_t* p); /** - * @brief Get the type of a type erased device array + * @brief Get the type of a type erased device array view * - * @param [in] p Pointer to the type erased device array + * @param [in] p Pointer to the type erased device array view * @return The type of the elements in the array */ -data_type_id_t cugraph_type_erased_device_array_type(const cugraph_type_erased_device_array_t* p); +data_type_id_t cugraph_type_erased_device_array_view_type( + const cugraph_type_erased_device_array_view_t* p); /** - * @brief Get the raw pointer of the type erased device array + * @brief Get the raw pointer of the type erased device array view * - * @param [in] p Pointer to the type erased device array + * @param [in] p Pointer to the type erased device array view * @return Pointer (device memory) for the data in the array */ -const void* cugraph_type_erased_device_array_pointer(const cugraph_type_erased_device_array_t* p); +const void* cugraph_type_erased_device_array_view_pointer( + const cugraph_type_erased_device_array_view_t* p); /** * @brief Create a type erased host array * * @param [in] handle Handle for accessing resources - * @param [in] dtype The type of array to create * @param [in] n_elems The number of elements in the array + * @param [in] dtype The type of array to create * @param [out] array Pointer to the location to store the pointer to the host array * @param [out] error Pointer to an error object storing details of any error. Will * be populated if error code is not CUGRAPH_SUCCESS * @return error code */ cugraph_error_code_t cugraph_type_erased_host_array_create(const cugraph_resource_handle_t* handle, - data_type_id_t dtype, size_t n_elems, + data_type_id_t dtype, cugraph_type_erased_host_array_t** array, cugraph_error_t** error); @@ -103,43 +154,84 @@ cugraph_error_code_t cugraph_type_erased_host_array_create(const cugraph_resourc */ void cugraph_type_erased_host_array_free(cugraph_type_erased_host_array_t* p); +#if 0 /** - * @brief Get the size of a type erased host array + * @brief Release the raw pointer of the type erased host array + * + * The caller is now responsible for freeing the host pointer * * @param [in] p Pointer to the type erased host array + * @return Pointer (host memory) for the data in the array + */ +void* cugraph_type_erased_host_array_release(cugraph_type_erased_host_array_t* p); +#endif + +/** + * @brief Create a type erased host array view from + * a type erased host array + * + * @param [in] array Pointer to the type erased host array + * @return Pointer to the view of the host array + */ +cugraph_type_erased_host_array_view_t* cugraph_type_erased_host_array_view( + cugraph_type_erased_host_array_t* array); + +/** + * @brief Create a type erased host array view from + * a raw host pointer. + * + * @param [in] pointer Raw host pointer + * @param [in] n_elems The number of elements in the array + * @param [in] dtype The type of array to create + * @return pointer to the view of the host array + */ +cugraph_type_erased_host_array_view_t* cugraph_type_erased_host_array_view_create( + void* pointer, size_t n_elems, data_type_id_t dtype); + +/** + * @brief Destroy a type erased host array view + * + * @param [in] p Pointer to the type erased host array view + */ +void cugraph_type_erased_host_array_view_free(cugraph_type_erased_host_array_view_t* p); + +/** + * @brief Get the size of a type erased host array view + * + * @param [in] p Pointer to the type erased host array view * @return The number of elements in the array */ -size_t cugraph_type_erased_host_array_size(const cugraph_type_erased_host_array_t* p); +size_t cugraph_type_erased_host_array_size(const cugraph_type_erased_host_array_view_t* p); /** - * @brief Get the type of a type erased host array + * @brief Get the type of a type erased host array view * - * @param [in] p Pointer to the type erased host array + * @param [in] p Pointer to the type erased host array view * @return The type of the elements in the array */ -data_type_id_t cugraph_type_erased_host_array_type(const cugraph_type_erased_host_array_t* p); +data_type_id_t cugraph_type_erased_host_array_type(const cugraph_type_erased_host_array_view_t* p); /** - * @brief Get the raw pointer of the type erased host array + * @brief Get the raw pointer of the type erased host array view * - * @param [in] p Pointer to the type erased host array + * @param [in] p Pointer to the type erased host array view * @return Pointer (host memory) for the data in the array */ -void* cugraph_type_erased_host_array_pointer(const cugraph_type_erased_host_array_t* p); +void* cugraph_type_erased_host_array_pointer(const cugraph_type_erased_host_array_view_t* p); /** - * @brief Copy data from host to a type erased device array + * @brief Copy data from host to a type erased device array view * * @param [in] handle Handle for accessing resources - * @param [out] dst Pointer to the type erased device array + * @param [out] dst Pointer to the type erased device array view * @param [in] h_src Pointer to host array to copy into device memory * @param [out] error Pointer to an error object storing details of any error. Will * be populated if error code is not CUGRAPH_SUCCESS * @return error code */ -cugraph_error_code_t cugraph_type_erased_device_array_copy_from_host( +cugraph_error_code_t cugraph_type_erased_device_array_view_copy_from_host( const cugraph_resource_handle_t* handle, - cugraph_type_erased_device_array_t* dst, + cugraph_type_erased_device_array_view_t* dst, const byte_t* h_src, cugraph_error_t** error); @@ -148,15 +240,31 @@ cugraph_error_code_t cugraph_type_erased_device_array_copy_from_host( * * @param [in] handle Handle for accessing resources * @param [out] h_dst Pointer to host array - * @param [in] src Pointer to the type erased device array to copy from + * @param [in] src Pointer to the type erased device array view source * @param [out] error Pointer to an error object storing details of any error. Will * be populated if error code is not CUGRAPH_SUCCESS * @return error code */ -cugraph_error_code_t cugraph_type_erased_device_array_copy_to_host( +cugraph_error_code_t cugraph_type_erased_device_array_view_copy_to_host( const cugraph_resource_handle_t* handle, byte_t* h_dst, - const cugraph_type_erased_device_array_t* src, + const cugraph_type_erased_device_array_view_t* src, + cugraph_error_t** error); + +/** + * @brief Copy data between two type erased device array views + * + * @param [in] handle Handle for accessing resources + * @param [out] dst Pointer to type erased device array view destination + * @param [in] src Pointer to type erased device array view source + * @param [out] error Pointer to an error object storing details of any error. Will + * be populated if error code is not CUGRAPH_SUCCESS + * @return error code + */ +cugraph_error_code_t cugraph_type_erased_device_array_view_copy( + const cugraph_resource_handle_t* handle, + cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* src, cugraph_error_t** error); #ifdef __cplusplus diff --git a/cpp/include/cugraph_c/graph.h b/cpp/include/cugraph_c/graph.h index 6860290ec82..ecb85b1852b 100644 --- a/cpp/include/cugraph_c/graph.h +++ b/cpp/include/cugraph_c/graph.h @@ -55,9 +55,9 @@ typedef struct { */ cugraph_error_code_t cugraph_sg_graph_create(const cugraph_resource_handle_t* handle, const cugraph_graph_properties_t* properties, - const cugraph_type_erased_device_array_t* src, - const cugraph_type_erased_device_array_t* dst, - const cugraph_type_erased_device_array_t* weights, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, bool_t store_transposed, bool_t renumber, bool_t check, @@ -97,11 +97,11 @@ void cugraph_sg_graph_free(cugraph_graph_t* graph); cugraph_error_code_t cugraph_mg_graph_create( const cugraph_resource_handle_t* handle, const cugraph_graph_properties_t* properties, - const cugraph_type_erased_device_array_t* src, - const cugraph_type_erased_device_array_t* dst, - const cugraph_type_erased_device_array_t* weights, - const cugraph_type_erased_host_array_t* vertex_partition_offsets, - const cugraph_type_erased_host_array_t* segment_offsets, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_host_array_view_t* vertex_partition_offsets, + const cugraph_type_erased_host_array_view_t* segment_offsets, bool_t store_transposed, size_t num_vertices, size_t num_edges, diff --git a/cpp/src/c_api/array.cpp b/cpp/src/c_api/array.cpp index 01253954b1e..47f2dd883b2 100644 --- a/cpp/src/c_api/array.cpp +++ b/cpp/src/c_api/array.cpp @@ -32,8 +32,8 @@ cugraph::visitors::DTypes dtypes_mapping[] = {cugraph::visitors::DTypes::INT32, extern "C" cugraph_error_code_t cugraph_type_erased_device_array_create( const cugraph_resource_handle_t* handle, - data_type_id_t dtype, size_t n_elems, + data_type_id_t dtype, cugraph_type_erased_device_array_t** array, cugraph_error_t** error) { @@ -68,33 +68,70 @@ extern "C" void cugraph_type_erased_device_array_free(cugraph_type_erased_device delete internal_pointer; } -extern "C" size_t cugraph_type_erased_device_array_size(const cugraph_type_erased_device_array_t* p) +#if 0 +// NOTE: This can't work. rmm::device_buffer doesn't support release, that would leave a raw +// pointer in the wild with no idea how to free it. I suppose that could be done +// (I imagine you can do that with unique_ptr), but it's not currently supported and I'm +// not sure *this* use case is sufficient justification to adding a potentially +// dangerous feature. +extern "C" void* cugraph_type_erased_device_array_release(cugraph_type_erased_device_array_t* p) +{ + auto internal_pointer = reinterpret_cast(p); + return internal_pointer->data_.release(); +} +#endif + +extern "C" cugraph_type_erased_device_array_view_t* cugraph_type_erased_device_array_view( + cugraph_type_erased_device_array_t* array) +{ + auto internal_pointer = + reinterpret_cast(array); + return reinterpret_cast(internal_pointer->view()); +} + +cugraph_type_erased_device_array_view_t* cugraph_type_erased_device_array_view_create( + void* pointer, size_t n_elems, data_type_id_t dtype) +{ + return reinterpret_cast( + new cugraph::c_api::cugraph_type_erased_device_array_view_t{pointer, n_elems, dtype}); +} + +extern "C" void cugraph_type_erased_device_array_view_free( + cugraph_type_erased_device_array_view_t* p) +{ + auto internal_pointer = + reinterpret_cast(p); + delete internal_pointer; +} + +extern "C" size_t cugraph_type_erased_device_array_view_size( + const cugraph_type_erased_device_array_view_t* p) { auto internal_pointer = - reinterpret_cast(p); + reinterpret_cast(p); return internal_pointer->size_; } -extern "C" data_type_id_t cugraph_type_erased_device_array_type( - const cugraph_type_erased_device_array_t* p) +extern "C" data_type_id_t cugraph_type_erased_device_array_view_type( + const cugraph_type_erased_device_array_view_t* p) { auto internal_pointer = - reinterpret_cast(p); + reinterpret_cast(p); return internal_pointer->type_; } -extern "C" const void* cugraph_type_erased_device_array_pointer( - const cugraph_type_erased_device_array_t* p) +extern "C" const void* cugraph_type_erased_device_array_view_pointer( + const cugraph_type_erased_device_array_view_t* p) { auto internal_pointer = - reinterpret_cast(p); - return internal_pointer->data_.data(); + reinterpret_cast(p); + return internal_pointer->data_; } extern "C" cugraph_error_code_t cugraph_type_erased_host_array_create( const cugraph_resource_handle_t* handle, - data_type_id_t dtype, size_t n_elems, + data_type_id_t dtype, cugraph_type_erased_host_array_t** array, cugraph_error_t** error) { @@ -112,11 +149,10 @@ extern "C" cugraph_error_code_t cugraph_type_erased_host_array_create( size_t n_bytes = n_elems * (::data_type_sz[dtype]); - cugraph::c_api::cugraph_type_erased_host_array_t* ret_value = + *array = reinterpret_cast( new cugraph::c_api::cugraph_type_erased_host_array_t{ - new std::byte[n_bytes], n_elems, n_bytes, dtype}; + std::make_unique(n_bytes), n_elems, n_bytes, dtype}); - *array = reinterpret_cast(ret_value); return CUGRAPH_SUCCESS; } catch (std::exception const& ex) { auto tmp_error = new cugraph::c_api::cugraph_error_t{ex.what()}; @@ -128,35 +164,68 @@ extern "C" cugraph_error_code_t cugraph_type_erased_host_array_create( extern "C" void cugraph_type_erased_host_array_free(cugraph_type_erased_host_array_t* p) { auto internal_pointer = reinterpret_cast(p); - delete[] internal_pointer->data_; delete internal_pointer; } -extern "C" size_t cugraph_type_erased_host_array_size(const cugraph_type_erased_host_array_t* p) +#if 0 +// Leaving this one out since we're not doing the more important device version +extern "C" void* cugraph_type_erased_host_array_release(const cugraph_type_erased_host_array_t* p) +{ + auto internal_pointer = reinterpret_cast(p); + return internal_pointer->data_.release(); +} +#endif + +extern "C" cugraph_type_erased_host_array_view_t* cugraph_type_erased_host_array_view( + cugraph_type_erased_host_array_t* array) +{ + auto internal_pointer = + reinterpret_cast(array); + return reinterpret_cast(internal_pointer->view()); +} + +extern "C" cugraph_type_erased_host_array_view_t* cugraph_type_erased_host_array_view_create( + void* pointer, size_t n_elems, data_type_id_t dtype) +{ + return reinterpret_cast( + new cugraph::c_api::cugraph_type_erased_host_array_view_t{ + static_cast(pointer), n_elems, dtype}); +} + +extern "C" void cugraph_type_erased_host_array_view_free(cugraph_type_erased_host_array_view_t* p) +{ + auto internal_pointer = + reinterpret_cast(p); + delete internal_pointer; +} + +extern "C" size_t cugraph_type_erased_host_array_size( + const cugraph_type_erased_host_array_view_t* p) { auto internal_pointer = - reinterpret_cast(p); + reinterpret_cast(p); return internal_pointer->size_; } -extern "C" data_type_id_t cugraph_type_erased_host_array_type( - const cugraph_type_erased_host_array_t* p) +extern "C" data_type_id_t cugraph_type_erased_host_array_view_type( + const cugraph_type_erased_host_array_view_t* p) { auto internal_pointer = - reinterpret_cast(p); + reinterpret_cast(p); return internal_pointer->type_; } -extern "C" void* cugraph_type_erased_host_array_pointer(const cugraph_type_erased_host_array_t* p) +extern "C" void* cugraph_type_erased_host_array_pointer( + const cugraph_type_erased_host_array_view_t* p) { auto internal_pointer = - reinterpret_cast(p); + reinterpret_cast(p); return internal_pointer->data_; } -extern "C" cugraph_error_code_t cugraph_type_erased_device_array_copy_from_host( +extern "C" cugraph_error_code_t cugraph_type_erased_device_array_view_copy_from_host( const cugraph_resource_handle_t* handle, - cugraph_type_erased_device_array_t* dst, + cugraph_type_erased_device_array_view_t* dst, const byte_t* h_src, cugraph_error_t** error) { @@ -165,7 +234,7 @@ extern "C" cugraph_error_code_t cugraph_type_erased_device_array_copy_from_host( try { raft::handle_t const* raft_handle = reinterpret_cast(handle); auto internal_pointer = - reinterpret_cast(dst); + reinterpret_cast(dst); if (!raft_handle) { *error = reinterpret_cast( @@ -173,9 +242,9 @@ extern "C" cugraph_error_code_t cugraph_type_erased_device_array_copy_from_host( return CUGRAPH_INVALID_HANDLE; } - raft::update_device(reinterpret_cast(internal_pointer->data_.data()), + raft::update_device(reinterpret_cast(internal_pointer->data_), h_src, - internal_pointer->data_.size(), + internal_pointer->num_bytes(), raft_handle->get_stream()); return CUGRAPH_SUCCESS; @@ -186,10 +255,10 @@ extern "C" cugraph_error_code_t cugraph_type_erased_device_array_copy_from_host( } } -extern "C" cugraph_error_code_t cugraph_type_erased_device_array_copy_to_host( +extern "C" cugraph_error_code_t cugraph_type_erased_device_array_view_copy_to_host( const cugraph_resource_handle_t* handle, byte_t* h_dst, - const cugraph_type_erased_device_array_t* src, + const cugraph_type_erased_device_array_view_t* src, cugraph_error_t** error) { *error = nullptr; @@ -197,7 +266,7 @@ extern "C" cugraph_error_code_t cugraph_type_erased_device_array_copy_to_host( try { raft::handle_t const* raft_handle = reinterpret_cast(handle); auto internal_pointer = - reinterpret_cast(src); + reinterpret_cast(src); if (!raft_handle) { *error = reinterpret_cast( @@ -206,8 +275,48 @@ extern "C" cugraph_error_code_t cugraph_type_erased_device_array_copy_to_host( } raft::update_host(h_dst, - reinterpret_cast(internal_pointer->data_.data()), - internal_pointer->data_.size(), + reinterpret_cast(internal_pointer->data_), + internal_pointer->num_bytes(), + raft_handle->get_stream()); + + return CUGRAPH_SUCCESS; + } catch (std::exception const& ex) { + auto tmp_error = new cugraph::c_api::cugraph_error_t{ex.what()}; + *error = reinterpret_cast(tmp_error); + return CUGRAPH_UNKNOWN_ERROR; + } +} + +extern "C" cugraph_error_code_t cugraph_type_erased_device_array_view_copy( + const cugraph_resource_handle_t* handle, + cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* src, + cugraph_error_t** error) +{ + *error = nullptr; + + try { + raft::handle_t const* raft_handle = reinterpret_cast(handle); + auto internal_pointer_dst = + reinterpret_cast(dst); + auto internal_pointer_src = + reinterpret_cast(src); + + if (!raft_handle) { + *error = reinterpret_cast( + new cugraph::c_api::cugraph_error_t{"invalid resource handle"}); + return CUGRAPH_INVALID_HANDLE; + } + + if (internal_pointer_src->num_bytes() != internal_pointer_dst->num_bytes()) { + *error = reinterpret_cast( + new cugraph::c_api::cugraph_error_t{"source and destination arrays are different sizes"}); + return CUGRAPH_INVALID_INPUT; + } + + raft::update_host(reinterpret_cast(internal_pointer_dst->data_), + reinterpret_cast(internal_pointer_src->data_), + internal_pointer_src->num_bytes(), raft_handle->get_stream()); return CUGRAPH_SUCCESS; diff --git a/cpp/src/c_api/array.hpp b/cpp/src/c_api/array.hpp index 2df54fe7d30..e6a8d7d5b1a 100644 --- a/cpp/src/c_api/array.hpp +++ b/cpp/src/c_api/array.hpp @@ -27,16 +27,39 @@ namespace c_api { extern cugraph::visitors::DTypes dtypes_mapping[data_type_id_t::NTYPES]; +struct cugraph_type_erased_device_array_view_t { + void* data_; + size_t size_; + size_t num_bytes_; + data_type_id_t type_; + + template + T* as_type() + { + return reinterpret_cast(data_); + } + + template + T const* as_type() const + { + return reinterpret_cast(data_); + } + + size_t num_bytes() const { return num_bytes_; } +}; + struct cugraph_type_erased_device_array_t { + // NOTE: size must be first here because the device buffer is released size_t size_; + // Why doesn't rmm::device_buffer support release? rmm::device_buffer data_; data_type_id_t type_; cugraph_type_erased_device_array_t(size_t size, - size_t nbytes, + size_t num_bytes, data_type_id_t type, rmm::cuda_stream_view const& stream_view) - : data_(nbytes, stream_view), size_(size), type_(type) + : size_(size), data_(num_bytes, stream_view), type_(type) { } @@ -46,23 +69,16 @@ struct cugraph_type_erased_device_array_t { { } - template - T* as_type() - { - return reinterpret_cast(data_.data()); - } - - template - T const* as_type() const + auto view() { - return reinterpret_cast(data_.data()); + return new cugraph_type_erased_device_array_view_t{data_.data(), size_, data_.size(), type_}; } }; -struct cugraph_type_erased_host_array_t { +struct cugraph_type_erased_host_array_view_t { std::byte* data_; size_t size_; - size_t nbytes_; + size_t num_bytes_; data_type_id_t type_; template @@ -70,6 +86,17 @@ struct cugraph_type_erased_host_array_t { { return reinterpret_cast(data_); } + + size_t num_bytes() const { return num_bytes_; } +}; + +struct cugraph_type_erased_host_array_t { + std::unique_ptr data_; + size_t size_; + size_t num_bytes_; + data_type_id_t type_; + + auto view() { return new cugraph_type_erased_host_array_view_t{data_.get(), size_, num_bytes_, type_}; } }; } // namespace c_api diff --git a/cpp/src/c_api/bfs.cpp b/cpp/src/c_api/bfs.cpp index f5da8c7ff74..c4b28cabb65 100644 --- a/cpp/src/c_api/bfs.cpp +++ b/cpp/src/c_api/bfs.cpp @@ -33,7 +33,7 @@ namespace c_api { struct bfs_functor : public abstract_functor { raft::handle_t const& handle_; cugraph_graph_t* graph_; - cugraph_type_erased_device_array_t* sources_; + cugraph_type_erased_device_array_view_t* sources_; bool direction_optimizing_; size_t depth_limit_; bool do_expensive_check_; @@ -42,7 +42,7 @@ struct bfs_functor : public abstract_functor { bfs_functor(raft::handle_t const& handle, cugraph_graph_t* graph, - cugraph_type_erased_device_array_t* sources, + cugraph_type_erased_device_array_view_t* sources, bool direction_optimizing, size_t depth_limit, bool do_expensive_check, @@ -141,25 +141,25 @@ struct bfs_functor : public abstract_functor { } // namespace c_api } // namespace cugraph -extern "C" cugraph_type_erased_device_array_t* cugraph_paths_result_get_vertices( +extern "C" cugraph_type_erased_device_array_view_t* cugraph_paths_result_get_vertices( cugraph_paths_result_t* result) { auto internal_pointer = reinterpret_cast(result); - return reinterpret_cast(internal_pointer->vertex_ids_); + return reinterpret_cast(internal_pointer->vertex_ids_->view()); } -extern "C" cugraph_type_erased_device_array_t* cugraph_paths_result_get_distances( +extern "C" cugraph_type_erased_device_array_view_t* cugraph_paths_result_get_distances( cugraph_paths_result_t* result) { auto internal_pointer = reinterpret_cast(result); - return reinterpret_cast(internal_pointer->distances_); + return reinterpret_cast(internal_pointer->distances_->view()); } -extern "C" cugraph_type_erased_device_array_t* cugraph_paths_result_get_predecessors( +extern "C" cugraph_type_erased_device_array_view_t* cugraph_paths_result_get_predecessors( cugraph_paths_result_t* result) { auto internal_pointer = reinterpret_cast(result); - return reinterpret_cast(internal_pointer->predecessors_); + return reinterpret_cast(internal_pointer->predecessors_->view()); } extern "C" void cugraph_paths_result_free(cugraph_paths_result_t* result) @@ -173,7 +173,7 @@ extern "C" void cugraph_paths_result_free(cugraph_paths_result_t* result) extern "C" cugraph_error_code_t cugraph_bfs(const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, - cugraph_type_erased_device_array_t* sources, + cugraph_type_erased_device_array_view_t* sources, bool_t direction_optimizing, size_t depth_limit, bool_t do_expensive_check, @@ -187,7 +187,7 @@ extern "C" cugraph_error_code_t cugraph_bfs(const cugraph_resource_handle_t* han try { auto p_handle = reinterpret_cast(handle); auto p_graph = reinterpret_cast(graph); - auto p_sources = reinterpret_cast(sources); + auto p_sources = reinterpret_cast(sources); cugraph::c_api::bfs_functor functor(*p_handle, p_graph, diff --git a/cpp/src/c_api/extract_paths.cpp b/cpp/src/c_api/extract_paths.cpp index 5f6cd6885c5..c9bf88033c2 100644 --- a/cpp/src/c_api/extract_paths.cpp +++ b/cpp/src/c_api/extract_paths.cpp @@ -38,16 +38,16 @@ struct cugraph_extract_paths_result_t { struct extract_paths_functor : public abstract_functor { raft::handle_t const& handle_; cugraph_graph_t* graph_; - cugraph_type_erased_device_array_t const* sources_; + cugraph_type_erased_device_array_view_t const* sources_; cugraph_paths_result_t const* paths_result_; - cugraph_type_erased_device_array_t const* destinations_; + cugraph_type_erased_device_array_view_t const* destinations_; cugraph_extract_paths_result_t* result_{}; extract_paths_functor(raft::handle_t const& handle, cugraph_graph_t* graph, - cugraph_type_erased_device_array_t const* sources, + cugraph_type_erased_device_array_view_t const* sources, cugraph_paths_result_t const* paths_result, - cugraph_type_erased_device_array_t const* destinations) + cugraph_type_erased_device_array_view_t const* destinations) : abstract_functor(), handle_(handle), graph_(graph), @@ -93,8 +93,8 @@ struct extract_paths_functor : public abstract_functor { rmm::device_uvector predecessors(paths_result_->predecessors_->size_, handle_.get_stream()); raft::copy(predecessors.data(), - paths_result_->predecessors_->as_type(), - paths_result_->predecessors_->size_, + paths_result_->predecessors_->view()->as_type(), + paths_result_->predecessors_->view()->size_, handle_.get_stream()); // @@ -120,7 +120,7 @@ struct extract_paths_functor : public abstract_functor { cugraph::extract_bfs_paths( handle_, graph_view, - paths_result_->distances_->as_type(), + paths_result_->distances_->view()->as_type(), predecessors.data(), destinations.data(), destinations.size()); @@ -147,11 +147,11 @@ extern "C" size_t cugraph_extract_paths_result_get_max_path_length( return internal_pointer->max_path_length_; } -cugraph_type_erased_device_array_t* cugraph_extract_paths_result_get_paths( +cugraph_type_erased_device_array_view_t* cugraph_extract_paths_result_get_paths( cugraph_extract_paths_result_t* result) { auto internal_pointer = reinterpret_cast(result); - return reinterpret_cast(internal_pointer->paths_); + return reinterpret_cast(internal_pointer->paths_); } extern "C" void cugraph_extract_paths_result_free(cugraph_extract_paths_result_t* result) @@ -164,9 +164,9 @@ extern "C" void cugraph_extract_paths_result_free(cugraph_extract_paths_result_t extern "C" cugraph_error_code_t cugraph_extract_paths( const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, - const cugraph_type_erased_device_array_t* sources, + const cugraph_type_erased_device_array_view_t* sources, const cugraph_paths_result_t* paths_result, - const cugraph_type_erased_device_array_t* destinations, + const cugraph_type_erased_device_array_view_t* destinations, cugraph_extract_paths_result_t** result, cugraph_error_t** error) { @@ -177,11 +177,11 @@ extern "C" cugraph_error_code_t cugraph_extract_paths( auto p_handle = reinterpret_cast(handle); auto p_graph = reinterpret_cast(graph); auto p_sources = - reinterpret_cast(sources); + reinterpret_cast(sources); auto p_paths_result = reinterpret_cast(paths_result); auto p_destinations = - reinterpret_cast(destinations); + reinterpret_cast(destinations); cugraph::c_api::extract_paths_functor functor( *p_handle, p_graph, p_sources, p_paths_result, p_destinations); diff --git a/cpp/src/c_api/graph_mg.cpp b/cpp/src/c_api/graph_mg.cpp index 9fc3d9ba7af..6b8538f0640 100644 --- a/cpp/src/c_api/graph_mg.cpp +++ b/cpp/src/c_api/graph_mg.cpp @@ -19,11 +19,11 @@ extern "C" cugraph_error_code_t cugraph_mg_graph_create( const cugraph_resource_handle_t* handle, const cugraph_graph_properties_t* properties, - const cugraph_type_erased_device_array_t* src, - const cugraph_type_erased_device_array_t* dst, - const cugraph_type_erased_device_array_t* weights, - const cugraph_type_erased_host_array_t* vertex_partition_offsets, - const cugraph_type_erased_host_array_t* segment_offsets, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_host_array_view_t* vertex_partition_offsets, + const cugraph_type_erased_host_array_view_t* segment_offsets, bool_t store_transposed, size_t num_vertices, size_t num_edges, diff --git a/cpp/src/c_api/graph_sg.cpp b/cpp/src/c_api/graph_sg.cpp index e51e94edeeb..c06cd241e9d 100644 --- a/cpp/src/c_api/graph_sg.cpp +++ b/cpp/src/c_api/graph_sg.cpp @@ -32,9 +32,9 @@ namespace c_api { struct create_graph_functor : public abstract_functor { raft::handle_t const& handle_; cugraph_graph_properties_t const* properties_; - c_api::cugraph_type_erased_device_array_t const* src_; - c_api::cugraph_type_erased_device_array_t const* dst_; - c_api::cugraph_type_erased_device_array_t const* weights_; + c_api::cugraph_type_erased_device_array_view_t const* src_; + c_api::cugraph_type_erased_device_array_view_t const* dst_; + c_api::cugraph_type_erased_device_array_view_t const* weights_; bool_t renumber_; bool_t check_; data_type_id_t edge_type_; @@ -42,9 +42,9 @@ struct create_graph_functor : public abstract_functor { create_graph_functor(raft::handle_t const& handle, cugraph_graph_properties_t const* properties, - c_api::cugraph_type_erased_device_array_t const* src, - c_api::cugraph_type_erased_device_array_t const* dst, - c_api::cugraph_type_erased_device_array_t const* weights, + c_api::cugraph_type_erased_device_array_view_t const* src, + c_api::cugraph_type_erased_device_array_view_t const* dst, + c_api::cugraph_type_erased_device_array_view_t const* weights, bool_t renumber, bool_t check, data_type_id_t edge_type) @@ -173,9 +173,9 @@ struct destroy_graph_functor : public abstract_functor { extern "C" cugraph_error_code_t cugraph_sg_graph_create( const cugraph_resource_handle_t* handle, const cugraph_graph_properties_t* properties, - const cugraph_type_erased_device_array_t* src, - const cugraph_type_erased_device_array_t* dst, - const cugraph_type_erased_device_array_t* weights, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, bool_t store_transposed, bool_t renumber, bool_t check, @@ -189,10 +189,10 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( *error = nullptr; auto p_handle = reinterpret_cast(handle); - auto p_src = reinterpret_cast(src); - auto p_dst = reinterpret_cast(dst); + auto p_src = reinterpret_cast(src); + auto p_dst = reinterpret_cast(dst); auto p_weights = - reinterpret_cast(weights); + reinterpret_cast(weights); CAPI_EXPECTS(p_src->size_ == p_dst->size_, CUGRAPH_INVALID_INPUT, @@ -202,6 +202,7 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( CUGRAPH_INVALID_INPUT, "Invalid input arguments: src type != dst type.", *error); + CAPI_EXPECTS(!weights || (p_weights->size_ == p_src->size_), CUGRAPH_INVALID_INPUT, "Invalid input arguments: src size != weights size.", diff --git a/cpp/src/c_api/pagerank.cpp b/cpp/src/c_api/pagerank.cpp index c309b3a4789..4eea0ae64a8 100644 --- a/cpp/src/c_api/pagerank.cpp +++ b/cpp/src/c_api/pagerank.cpp @@ -39,9 +39,9 @@ struct cugraph_pagerank_result_t { struct pagerank_functor : public abstract_functor { raft::handle_t const& handle_; cugraph_graph_t* graph_; - cugraph_type_erased_device_array_t const* precomputed_vertex_out_weight_sums_; - cugraph_type_erased_device_array_t* personalization_vertices_; - cugraph_type_erased_device_array_t const* personalization_values_; + cugraph_type_erased_device_array_view_t const* precomputed_vertex_out_weight_sums_; + cugraph_type_erased_device_array_view_t* personalization_vertices_; + cugraph_type_erased_device_array_view_t const* personalization_values_; double alpha_; double epsilon_; size_t max_iterations_; @@ -51,9 +51,9 @@ struct pagerank_functor : public abstract_functor { pagerank_functor(raft::handle_t const& handle, cugraph_graph_t* graph, - cugraph_type_erased_device_array_t const* precomputed_vertex_out_weight_sums, - cugraph_type_erased_device_array_t* personalization_vertices, - cugraph_type_erased_device_array_t const* personalization_values, + cugraph_type_erased_device_array_view_t const* precomputed_vertex_out_weight_sums, + cugraph_type_erased_device_array_view_t* personalization_vertices, + cugraph_type_erased_device_array_view_t const* personalization_values, double alpha, double epsilon, size_t max_iterations, @@ -151,18 +151,18 @@ struct pagerank_functor : public abstract_functor { } // namespace c_api } // namespace cugraph -extern "C" cugraph_type_erased_device_array_t* cugraph_pagerank_result_get_vertices( +extern "C" cugraph_type_erased_device_array_view_t* cugraph_pagerank_result_get_vertices( cugraph_pagerank_result_t* result) { auto internal_pointer = reinterpret_cast(result); - return reinterpret_cast(internal_pointer->vertex_ids_); + return reinterpret_cast(internal_pointer->vertex_ids_->view()); } -extern "C" cugraph_type_erased_device_array_t* cugraph_pagerank_result_get_pageranks( +extern "C" cugraph_type_erased_device_array_view_t* cugraph_pagerank_result_get_pageranks( cugraph_pagerank_result_t* result) { auto internal_pointer = reinterpret_cast(result); - return reinterpret_cast(internal_pointer->pageranks_); + return reinterpret_cast(internal_pointer->pageranks_->view()); } extern "C" void cugraph_pagerank_result_free(cugraph_pagerank_result_t* result) @@ -176,7 +176,7 @@ extern "C" void cugraph_pagerank_result_free(cugraph_pagerank_result_t* result) extern "C" cugraph_error_code_t cugraph_pagerank( const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, - const cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, + const cugraph_type_erased_device_array_view_t* precomputed_vertex_out_weight_sums, double alpha, double epsilon, size_t max_iterations, @@ -193,7 +193,7 @@ extern "C" cugraph_error_code_t cugraph_pagerank( auto p_graph = reinterpret_cast(graph); auto p_precomputed_vertex_out_weight_sums = - reinterpret_cast( + reinterpret_cast( precomputed_vertex_out_weight_sums); cugraph::c_api::pagerank_functor functor(*p_handle, @@ -231,9 +231,9 @@ extern "C" cugraph_error_code_t cugraph_pagerank( extern "C" cugraph_error_code_t cugraph_personalized_pagerank( const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, - const cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, - cugraph_type_erased_device_array_t* personalization_vertices, - const cugraph_type_erased_device_array_t* personalization_values, + const cugraph_type_erased_device_array_view_t* precomputed_vertex_out_weight_sums, + cugraph_type_erased_device_array_view_t* personalization_vertices, + const cugraph_type_erased_device_array_view_t* personalization_values, double alpha, double epsilon, size_t max_iterations, @@ -250,13 +250,13 @@ extern "C" cugraph_error_code_t cugraph_personalized_pagerank( auto p_graph = reinterpret_cast(graph); auto p_precomputed_vertex_out_weight_sums = - reinterpret_cast( + reinterpret_cast( precomputed_vertex_out_weight_sums); auto p_personalization_vertices = - reinterpret_cast( + reinterpret_cast( personalization_vertices); auto p_personalization_values = - reinterpret_cast( + reinterpret_cast( personalization_vertices); cugraph::c_api::pagerank_functor functor(*p_handle, diff --git a/cpp/tests/c_api/bfs_test.c b/cpp/tests/c_api/bfs_test.c index ddc45e70505..6146ab191dd 100644 --- a/cpp/tests/c_api/bfs_test.c +++ b/cpp/tests/c_api/bfs_test.c @@ -46,6 +46,7 @@ int generic_bfs_test(vertex_t* h_src, cugraph_graph_t* p_graph = NULL; cugraph_paths_result_t* p_result = NULL; cugraph_type_erased_device_array_t* p_sources = NULL; + cugraph_type_erased_device_array_view_t* p_source_view = NULL; p_handle = cugraph_create_resource_handle(); TEST_ASSERT(test_ret_value, p_handle != NULL, "resource handle creation failed."); @@ -54,15 +55,17 @@ int generic_bfs_test(vertex_t* h_src, p_handle, h_src, h_dst, h_wgt, num_edges, store_transposed, &p_graph, &ret_error); ret_code = - cugraph_type_erased_device_array_create(p_handle, INT32, num_seeds, &p_sources, &ret_error); + cugraph_type_erased_device_array_create(p_handle, num_seeds, INT32, &p_sources, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "p_sources create failed."); - ret_code = cugraph_type_erased_device_array_copy_from_host( - p_handle, p_sources, (byte_t*)h_seeds, &ret_error); + p_source_view = cugraph_type_erased_device_array_view(p_sources); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + p_handle, p_source_view, (byte_t*)h_seeds, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); ret_code = cugraph_bfs( - p_handle, p_graph, p_sources, FALSE, depth_limit, FALSE, TRUE, &p_result, &ret_error); + p_handle, p_graph, p_source_view, FALSE, depth_limit, FALSE, TRUE, &p_result, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_bfs failed."); cugraph_type_erased_device_array_t* vertices; @@ -77,15 +80,15 @@ int generic_bfs_test(vertex_t* h_src, vertex_t h_distances[num_vertices]; vertex_t h_predecessors[num_vertices]; - ret_code = cugraph_type_erased_device_array_copy_to_host( + ret_code = cugraph_type_erased_device_array_view_copy_to_host( p_handle, (byte_t*)h_vertices, vertices, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); - ret_code = cugraph_type_erased_device_array_copy_to_host( + ret_code = cugraph_type_erased_device_array_view_copy_to_host( p_handle, (byte_t*)h_distances, distances, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); - ret_code = cugraph_type_erased_device_array_copy_to_host( + ret_code = cugraph_type_erased_device_array_view_copy_to_host( p_handle, (byte_t*)h_predecessors, predecessors, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); diff --git a/cpp/tests/c_api/create_graph_test.c b/cpp/tests/c_api/create_graph_test.c index e4368fe2f5a..63837b52074 100644 --- a/cpp/tests/c_api/create_graph_test.c +++ b/cpp/tests/c_api/create_graph_test.c @@ -56,31 +56,38 @@ int test_create_sg_graph_simple() cugraph_type_erased_device_array_t* src; cugraph_type_erased_device_array_t* dst; cugraph_type_erased_device_array_t* wgt; + cugraph_type_erased_device_array_view_t* src_view; + cugraph_type_erased_device_array_view_t* dst_view; + cugraph_type_erased_device_array_view_t* wgt_view; - ret_code = cugraph_type_erased_device_array_create(p_handle, vertex_tid, num_edges, &src, &ret_error); + ret_code = cugraph_type_erased_device_array_create(p_handle, num_edges, vertex_tid, &src, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); - ret_code = cugraph_type_erased_device_array_create(p_handle, vertex_tid, num_edges, &dst, &ret_error); + ret_code = cugraph_type_erased_device_array_create(p_handle, num_edges, vertex_tid, &dst, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); - ret_code = cugraph_type_erased_device_array_create(p_handle, weight_tid, num_edges, &wgt, &ret_error); + ret_code = cugraph_type_erased_device_array_create(p_handle, num_edges, weight_tid, &wgt, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); - ret_code = cugraph_type_erased_device_array_copy_from_host(p_handle, src, (byte_t*)h_src, &ret_error); + src_view = cugraph_type_erased_device_array_view(src); + dst_view = cugraph_type_erased_device_array_view(dst); + wgt_view = cugraph_type_erased_device_array_view(wgt); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host(p_handle, src_view, (byte_t*)h_src, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); - ret_code = cugraph_type_erased_device_array_copy_from_host(p_handle, dst, (byte_t*)h_dst, &ret_error); + ret_code = cugraph_type_erased_device_array_view_copy_from_host(p_handle, dst_view, (byte_t*)h_dst, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst copy_from_host failed."); - ret_code = cugraph_type_erased_device_array_copy_from_host(p_handle, wgt, (byte_t*)h_wgt, &ret_error); + ret_code = cugraph_type_erased_device_array_view_copy_from_host(p_handle, wgt_view, (byte_t*)h_wgt, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); ret_code = cugraph_sg_graph_create(p_handle, &properties, - src, - dst, - wgt, + src_view, + dst_view, + wgt_view, FALSE, FALSE, FALSE, @@ -90,10 +97,11 @@ int test_create_sg_graph_simple() cugraph_sg_graph_free(p_graph); + cugraph_type_erased_device_array_view_free(wgt_view); + cugraph_type_erased_device_array_view_free(dst_view); + cugraph_type_erased_device_array_view_free(src_view); cugraph_type_erased_device_array_free(wgt); - cugraph_type_erased_device_array_free(dst); - cugraph_type_erased_device_array_free(src); cugraph_free_resource_handle(p_handle); diff --git a/cpp/tests/c_api/extract_paths_test.c b/cpp/tests/c_api/extract_paths_test.c index d618d085c87..8e95807e2b9 100644 --- a/cpp/tests/c_api/extract_paths_test.c +++ b/cpp/tests/c_api/extract_paths_test.c @@ -44,12 +44,14 @@ int generic_bfs_test_with_extract_paths(vertex_t* h_src, cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; cugraph_error_t* ret_error; - cugraph_resource_handle_t* p_handle = NULL; - cugraph_graph_t* p_graph = NULL; - cugraph_paths_result_t* p_paths_result = NULL; - cugraph_extract_paths_result_t* p_extract_paths_result = NULL; - cugraph_type_erased_device_array_t* p_sources = NULL; - cugraph_type_erased_device_array_t* p_destinations = NULL; + cugraph_resource_handle_t* p_handle = NULL; + cugraph_graph_t* p_graph = NULL; + cugraph_paths_result_t* p_paths_result = NULL; + cugraph_extract_paths_result_t* p_extract_paths_result = NULL; + cugraph_type_erased_device_array_t* p_sources = NULL; + cugraph_type_erased_device_array_t* p_destinations = NULL; + cugraph_type_erased_device_array_view_t* p_sources_view = NULL; + cugraph_type_erased_device_array_view_t* p_destinations_view = NULL; p_handle = cugraph_create_resource_handle(); TEST_ASSERT(test_ret_value, p_handle != NULL, "resource handle creation failed."); @@ -58,30 +60,41 @@ int generic_bfs_test_with_extract_paths(vertex_t* h_src, p_handle, h_src, h_dst, h_wgt, num_edges, store_transposed, &p_graph, &ret_error); ret_code = - cugraph_type_erased_device_array_create(p_handle, INT32, num_seeds, &p_sources, &ret_error); + cugraph_type_erased_device_array_create(p_handle, num_seeds, INT32, &p_sources, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "p_sources create failed."); - ret_code = cugraph_type_erased_device_array_copy_from_host( - p_handle, p_sources, (byte_t*)h_seeds, &ret_error); + p_sources_view = cugraph_type_erased_device_array_view(p_sources); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + p_handle, p_sources_view, (byte_t*)h_seeds, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); ret_code = cugraph_type_erased_device_array_create( - p_handle, INT32, num_destinations, &p_destinations, &ret_error); + p_handle, num_destinations, INT32, &p_destinations, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "p_destinations create failed."); - ret_code = cugraph_type_erased_device_array_copy_from_host( - p_handle, p_destinations, (byte_t*)h_destinations, &ret_error); + p_destinations_view = cugraph_type_erased_device_array_view(p_destinations); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + p_handle, p_destinations_view, (byte_t*)h_destinations, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); - ret_code = cugraph_bfs( - p_handle, p_graph, p_sources, FALSE, depth_limit, FALSE, TRUE, &p_paths_result, &ret_error); + ret_code = cugraph_bfs(p_handle, + p_graph, + p_sources_view, + FALSE, + depth_limit, + FALSE, + TRUE, + &p_paths_result, + &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_bfs failed."); ret_code = cugraph_extract_paths(p_handle, p_graph, - p_sources, + p_sources_view, p_paths_result, - p_destinations, + p_destinations_view, &p_extract_paths_result, &ret_error); @@ -89,19 +102,27 @@ int generic_bfs_test_with_extract_paths(vertex_t* h_src, cugraph_type_erased_device_array_t* paths = cugraph_extract_paths_result_get_paths(p_extract_paths_result); - size_t paths_size = cugraph_type_erased_device_array_size(paths); + cugraph_type_erased_device_array_view_t* paths_view = + cugraph_type_erased_device_array_view(paths); + + size_t paths_size = cugraph_type_erased_device_array_view_size(paths_view); + vertex_t h_paths[paths_size]; - ret_code = - cugraph_type_erased_device_array_copy_to_host(p_handle, (byte_t*)h_paths, paths, &ret_error); + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + p_handle, (byte_t*)h_paths, paths_view, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); for (int i = 0; (i < paths_size) && (test_ret_value == 0); ++i) { TEST_ASSERT(test_ret_value, expected_paths[i] == h_paths[i], "paths don't match"); } + cugraph_type_erased_device_array_view_free(p_sources_view); + cugraph_type_erased_device_array_view_free(p_destinations_view); + cugraph_type_erased_device_array_view_free(paths_view); cugraph_type_erased_device_array_free(p_sources); cugraph_type_erased_device_array_free(p_destinations); + cugraph_type_erased_device_array_free(paths); cugraph_extract_paths_result_free(p_extract_paths_result); cugraph_paths_result_free(p_paths_result); cugraph_sg_graph_free(p_graph); diff --git a/cpp/tests/c_api/pagerank_test.c b/cpp/tests/c_api/pagerank_test.c index fb2faa210f7..144c26244c6 100644 --- a/cpp/tests/c_api/pagerank_test.c +++ b/cpp/tests/c_api/pagerank_test.c @@ -52,13 +52,14 @@ int generic_pagerank_test(vertex_t* h_src, p_handle, h_src, h_dst, h_wgt, num_edges, store_transposed, &p_graph, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "create_test_graph failed."); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); ret_code = cugraph_pagerank( p_handle, p_graph, NULL, alpha, epsilon, max_iterations, FALSE, FALSE, &p_result, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_pagerank failed."); - cugraph_type_erased_device_array_t* vertices; - cugraph_type_erased_device_array_t* pageranks; + cugraph_type_erased_device_array_view_t* vertices; + cugraph_type_erased_device_array_view_t* pageranks; vertices = cugraph_pagerank_result_get_vertices(p_result); pageranks = cugraph_pagerank_result_get_pageranks(p_result); @@ -66,11 +67,11 @@ int generic_pagerank_test(vertex_t* h_src, vertex_t h_vertices[num_vertices]; weight_t h_pageranks[num_vertices]; - ret_code = cugraph_type_erased_device_array_copy_to_host( + ret_code = cugraph_type_erased_device_array_view_copy_to_host( p_handle, (byte_t*)h_vertices, vertices, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); - ret_code = cugraph_type_erased_device_array_copy_to_host( + ret_code = cugraph_type_erased_device_array_view_copy_to_host( p_handle, (byte_t*)h_pageranks, pageranks, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); @@ -80,6 +81,8 @@ int generic_pagerank_test(vertex_t* h_src, "pagerank results don't match"); } + cugraph_type_erased_device_array_view_free(pageranks); + cugraph_type_erased_device_array_view_free(vertices); cugraph_pagerank_result_free(p_result); cugraph_sg_graph_free(p_graph); cugraph_free_resource_handle(p_handle); diff --git a/cpp/tests/c_api/test_utils.c b/cpp/tests/c_api/test_utils.c index 7cd9324b1f8..befaff6cef7 100644 --- a/cpp/tests/c_api/test_utils.c +++ b/cpp/tests/c_api/test_utils.c @@ -53,36 +53,54 @@ int create_test_graph(const cugraph_resource_handle_t* p_handle, cugraph_type_erased_device_array_t* src; cugraph_type_erased_device_array_t* dst; cugraph_type_erased_device_array_t* wgt; + cugraph_type_erased_device_array_view_t* src_view; + cugraph_type_erased_device_array_view_t* dst_view; + cugraph_type_erased_device_array_view_t* wgt_view; ret_code = - cugraph_type_erased_device_array_create(p_handle, vertex_tid, num_edges, &src, ret_error); + cugraph_type_erased_device_array_create(p_handle, num_edges, vertex_tid, &src, ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src create failed."); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(*ret_error)); ret_code = - cugraph_type_erased_device_array_create(p_handle, vertex_tid, num_edges, &dst, ret_error); + cugraph_type_erased_device_array_create(p_handle, num_edges, vertex_tid, &dst, ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst create failed."); ret_code = - cugraph_type_erased_device_array_create(p_handle, weight_tid, num_edges, &wgt, ret_error); + cugraph_type_erased_device_array_create(p_handle, num_edges, weight_tid, &wgt, ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt create failed."); + src_view = cugraph_type_erased_device_array_view(src); + dst_view = cugraph_type_erased_device_array_view(dst); + wgt_view = cugraph_type_erased_device_array_view(wgt); + ret_code = - cugraph_type_erased_device_array_copy_from_host(p_handle, src, (byte_t*)h_src, ret_error); + cugraph_type_erased_device_array_view_copy_from_host(p_handle, src_view, (byte_t*)h_src, ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); ret_code = - cugraph_type_erased_device_array_copy_from_host(p_handle, dst, (byte_t*)h_dst, ret_error); + cugraph_type_erased_device_array_view_copy_from_host(p_handle, dst_view, (byte_t*)h_dst, ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "dst copy_from_host failed."); ret_code = - cugraph_type_erased_device_array_copy_from_host(p_handle, wgt, (byte_t*)h_wgt, ret_error); + cugraph_type_erased_device_array_view_copy_from_host(p_handle, wgt_view, (byte_t*)h_wgt, ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "wgt copy_from_host failed."); - ret_code = cugraph_sg_graph_create( - p_handle, &properties, src, dst, wgt, store_transposed, FALSE, FALSE, p_graph, ret_error); + ret_code = cugraph_sg_graph_create(p_handle, + &properties, + src_view, + dst_view, + wgt_view, + store_transposed, + FALSE, + FALSE, + p_graph, + ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "graph creation failed."); + cugraph_type_erased_device_array_view_free(wgt_view); + cugraph_type_erased_device_array_view_free(dst_view); + cugraph_type_erased_device_array_view_free(src_view); cugraph_type_erased_device_array_free(wgt); cugraph_type_erased_device_array_free(dst); cugraph_type_erased_device_array_free(src); From d25eefe826b8bd21f898a4092426d2a3de606ff5 Mon Sep 17 00:00:00 2001 From: Chuck Hastings Date: Fri, 21 Jan 2022 20:36:15 -0500 Subject: [PATCH 17/43] update copyright --- cpp/include/cugraph_c/algorithms.h | 2 +- cpp/include/cugraph_c/graph.h | 2 +- cpp/src/c_api/array.cpp | 2 +- cpp/src/c_api/array.hpp | 2 +- cpp/src/c_api/bfs.cpp | 2 +- cpp/src/c_api/extract_paths.cpp | 2 +- cpp/src/c_api/graph_mg.cpp | 2 +- cpp/src/c_api/graph_sg.cpp | 2 +- cpp/src/c_api/pagerank.cpp | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/include/cugraph_c/algorithms.h b/cpp/include/cugraph_c/algorithms.h index 11923647cb5..3a46b88b4b8 100644 --- a/cpp/include/cugraph_c/algorithms.h +++ b/cpp/include/cugraph_c/algorithms.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp/include/cugraph_c/graph.h b/cpp/include/cugraph_c/graph.h index ecb85b1852b..6c64c317d17 100644 --- a/cpp/include/cugraph_c/graph.h +++ b/cpp/include/cugraph_c/graph.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp/src/c_api/array.cpp b/cpp/src/c_api/array.cpp index 47f2dd883b2..2ec6d1cb521 100644 --- a/cpp/src/c_api/array.cpp +++ b/cpp/src/c_api/array.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp/src/c_api/array.hpp b/cpp/src/c_api/array.hpp index e6a8d7d5b1a..c0e9ccbb3d1 100644 --- a/cpp/src/c_api/array.hpp +++ b/cpp/src/c_api/array.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp/src/c_api/bfs.cpp b/cpp/src/c_api/bfs.cpp index c4b28cabb65..a9fe7f4e5f7 100644 --- a/cpp/src/c_api/bfs.cpp +++ b/cpp/src/c_api/bfs.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp/src/c_api/extract_paths.cpp b/cpp/src/c_api/extract_paths.cpp index c9bf88033c2..ba2594c23a7 100644 --- a/cpp/src/c_api/extract_paths.cpp +++ b/cpp/src/c_api/extract_paths.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp/src/c_api/graph_mg.cpp b/cpp/src/c_api/graph_mg.cpp index 6b8538f0640..b8027bc1b37 100644 --- a/cpp/src/c_api/graph_mg.cpp +++ b/cpp/src/c_api/graph_mg.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp/src/c_api/graph_sg.cpp b/cpp/src/c_api/graph_sg.cpp index c06cd241e9d..f744ccbef18 100644 --- a/cpp/src/c_api/graph_sg.cpp +++ b/cpp/src/c_api/graph_sg.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp/src/c_api/pagerank.cpp b/cpp/src/c_api/pagerank.cpp index 4eea0ae64a8..74c0309092d 100644 --- a/cpp/src/c_api/pagerank.cpp +++ b/cpp/src/c_api/pagerank.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 8f8c1c9eb8ea7eb04cf8e520cfca9e9650e2b5f4 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Fri, 21 Jan 2022 22:15:57 -0600 Subject: [PATCH 18/43] Updated pagerank test to include expected results, added comments. --- .../pylibcugraph/_cugraph_c/graphs.pyx | 3 + .../pylibcugraph/_cugraph_c/pagerank.pyx | 19 ++++ .../pylibcugraph/tests/conftest.py | 86 +++++++++-------- .../tests/test_connected_components.py | 5 +- .../pylibcugraph/tests/test_graph_sg.py | 11 ++- .../pylibcugraph/tests/test_pagerank.py | 94 +++++++++++++++---- 6 files changed, 153 insertions(+), 65 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx index a7e528bf559..98eb10c4109 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx @@ -96,6 +96,7 @@ cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): "cugraph_type_erased_device_array_create()") # FIXME: add call to to device-device copy of __cuda_array_interface__ # values to cugraph_type_erased_device_array + # FIXME: don't do above fixme, use device view API # FIXME: set dtype properly error_code = cugraph_type_erased_device_array_create( @@ -109,6 +110,7 @@ cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): "cugraph_type_erased_device_array_create()") # FIXME: add call to to device-device copy of __cuda_array_interface__ # values to cugraph_type_erased_device_array + # FIXME: don't do above fixme, use device view API # FIXME: set dtype properly error_code = cugraph_type_erased_device_array_create( @@ -122,6 +124,7 @@ cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): "cugraph_type_erased_device_array_create()") # FIXME: add call to to device-device copy of __cuda_array_interface__ # values to cugraph_type_erased_device_array + # FIXME: don't do above fixme, use device view API error_code = cugraph_sg_graph_create( resource_handle.c_resource_handle_ptr, diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx b/python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx index c79181f2402..fa1381a4455 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx @@ -74,3 +74,22 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, do_expensive_check, &result_ptr, &error_ptr) + + +""" + cdef cugraph_pagerank_result_t* my_result + cugraph_pagerank(... , &my_result, ...) + + cdef cugraph_type_erased_device_array_t* verts + cdef cugraph_type_erased_device_array_t* prs + verts = cugraph_pagerank_result_get_vertices(my_result) + prs = cugraph_pagerank_result_get_pageranks(my_result) + + do device-device copy on verts to user array + do device-device copy on prs to user array +/* +size_t cugraph_type_erased_device_array_size(const cugraph_type_erased_device_array_t* p); +data_type_id_t cugraph_type_erased_device_array_type(const cugraph_type_erased_device_array_t* p); +const void* cugraph_type_erased_device_array_pointer(const cugraph_type_erased_device_array_t* p); +*/ +""" diff --git a/python/pylibcugraph/pylibcugraph/tests/conftest.py b/python/pylibcugraph/pylibcugraph/tests/conftest.py index ce668f66fae..28d4845b101 100644 --- a/python/pylibcugraph/pylibcugraph/tests/conftest.py +++ b/python/pylibcugraph/pylibcugraph/tests/conftest.py @@ -23,39 +23,43 @@ # ============================================================================= # Fixture parameters # ============================================================================= -class InlineGraphData: - @property - def name(self): - return self.__class__.__name__ - - @property - def is_valid(self): - return not(self.name.startswith("Invalid")) - - -class InvalidNumWeights_1(InlineGraphData): - srcs = cp.asarray([0, 1, 2], dtype=np.int32) - dsts = cp.asarray([1, 2, 3], dtype=np.int32) - weights = cp.asarray([0, 0, 0, 0], dtype=np.int32) - - -class InvalidNumVerts_1(InlineGraphData): - srcs = cp.asarray([1, 2], dtype=np.int32) - dsts = cp.asarray([1, 2, 3], dtype=np.int32) - weights = cp.asarray([0, 0, 0], dtype=np.int32) - - -class Simple_1(InlineGraphData): - srcs = cp.asarray([0, 1, 2], dtype=np.int32) - dsts = cp.asarray([1, 2, 3], dtype=np.int32) - weights = cp.asarray([0, 0, 0], dtype=np.int32) - - -class Simple_2(InlineGraphData): - srcs = cp.asarray([0, 1, 1, 2, 2, 2, 3, 4], dtype=np.int32) - dsts = cp.asarray([1, 3, 4, 0, 1, 3, 5, 5], dtype=np.int32) - weights = cp.asarray([0.1, 2.1, 1.1, 5.1, 3.1, 4.1, 7.2, 3.2], - dtype=np.float32) +class COOTestGraphDeviceData: + def __init__(self, srcs, dsts, weights, name): + self.srcs = srcs + self.dsts = dsts + self.weights = weights + self.name = name + self.is_valid = not(name.startswith("Invalid")) + + +InvalidNumWeights_1 = COOTestGraphDeviceData( + srcs=cp.asarray([0, 1, 2], dtype=np.int32), + dsts=cp.asarray([1, 2, 3], dtype=np.int32), + weights=cp.asarray([1.0, 1.0, 1.0, 1.0], dtype=np.float32), + name="InvalidNumWeights_1" + ) + +InvalidNumVerts_1 = COOTestGraphDeviceData( + srcs=cp.asarray([1, 2], dtype=np.int32), + dsts=cp.asarray([1, 2, 3], dtype=np.int32), + weights=cp.asarray([1.0, 1.0, 1.0], dtype=np.float32), + name="InvalidNumVerts_1" + ) + +Simple_1 = COOTestGraphDeviceData( + srcs=cp.asarray([0, 1, 2], dtype=np.int32), + dsts=cp.asarray([1, 2, 3], dtype=np.int32), + weights=cp.asarray([1.0, 1.0, 1.0], dtype=np.float32), + name="Simple_1" + ) + +Simple_2 = COOTestGraphDeviceData( + srcs=cp.asarray([0, 1, 1, 2, 2, 2, 3, 4], dtype=np.int32), + dsts=cp.asarray([1, 3, 4, 0, 1, 3, 5, 5], dtype=np.int32), + weights=cp.asarray([0.1, 2.1, 1.1, 5.1, 3.1, 4.1, 7.2, 3.2], + dtype=np.float32), + name="Simple_2" + ) # The objects in these lists must have a "name" attr, since fixtures will @@ -63,12 +67,12 @@ class Simple_2(InlineGraphData): # expected test results. The name attr is also used for the pytest test ID. valid_datasets = [utils.RAPIDS_DATASET_ROOT_DIR_PATH/"karate.csv", utils.RAPIDS_DATASET_ROOT_DIR_PATH/"dolphins.csv", - Simple_1(), - Simple_2(), + Simple_1, + Simple_2, ] all_datasets = valid_datasets + \ - [InvalidNumWeights_1(), - InvalidNumVerts_1(), + [InvalidNumWeights_1, + InvalidNumVerts_1, ] @@ -82,7 +86,7 @@ def get_graph_data_for_dataset(ds, ds_name): construct a graph object. The final value is a bool used to indicate if the data is valid or not (invalid to test error handling). """ - if isinstance(ds, InlineGraphData): + if isinstance(ds, COOTestGraphDeviceData): device_srcs = ds.srcs device_dsts = ds.dsts device_weights = ds.weights @@ -112,8 +116,8 @@ def graph_data(request): """ Return a series of cupy arrays that can be used to construct Graph objects. The parameterization includes invalid arrays which can be used to - error handling, so the final value returned indicated if the arrays are - valid or not. + test error handling, so the final value returned indicated if the arrays + are valid or not. """ return get_graph_data_for_dataset(request.param, request.param.name) @@ -123,7 +127,7 @@ def graph_data(request): def valid_graph_data(request): """ Return a series of cupy arrays that can be used to construct Graph objects, - all of which are valid. + all of which are valid (last value in returned tuple is always True). """ return get_graph_data_for_dataset(request.param, request.param.name) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_connected_components.py b/python/pylibcugraph/pylibcugraph/tests/test_connected_components.py index 7729b979bd1..9bef1e3b6df 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_connected_components.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_connected_components.py @@ -101,7 +101,7 @@ # ============================================================================= -# Pytest fixtures and helpers +# Pytest fixtures # ============================================================================= @pytest.fixture(scope="module", params=[pytest.param(value, id=key) @@ -145,6 +145,9 @@ def input_and_expected_output(request): expected_output_dict) +# ============================================================================= +# Helper functions +# ============================================================================= def _check_labels(vertex_ordered_labels, expected_vertex_comps): """ vertex_ordered_labels is a list of labels, ordered by the position of the diff --git a/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py b/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py index ad0195e92e1..73a2a984a7a 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py @@ -48,6 +48,10 @@ def test_graph_properties(): assert gp.is_symmetric is True assert gp.is_multigraph is True + gp = GraphProperties(is_multigraph=True, is_symmetric=False) + assert gp.is_symmetric is False + assert gp.is_multigraph is True + with pytest.raises(TypeError): gp = GraphProperties(is_symmetric="foo", is_multigraph=False) @@ -69,12 +73,11 @@ def test_sg_graph(graph_data): ResourceHandle, GraphProperties, ) - + # is_valid will only be True if the arrays are expected to produce a valid + # graph. If False, ensure SGGraph() raises the proper exception. (device_srcs, device_dsts, device_weights, ds_name, is_valid) = graph_data - graph_props = GraphProperties() - graph_props.is_symmetric = False - graph_props.is_multigraph = False + graph_props = GraphProperties(is_symmetric=False, is_multigraph=False) resource_handle = ResourceHandle() if is_valid: diff --git a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py index 41d3441401c..776211cffae 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py @@ -12,45 +12,101 @@ # limitations under the License. import pytest -# import cupy as cp -# import numpy as np +import cupy as cp +import numpy as np +# ============================================================================= +# Test data +# ============================================================================= +_alpha = 0.85 +_epsilon = 1.0e-6 +_max_iterations = 500 + +# Map the names of input data to expected pagerank output +# The result names correspond to the datasets defined in conftest.py +_test_data = {"karate.csv" : + (cp.asarray(range(34), dtype=np.int32), + cp.asarray( + [0.096998, 0.052877, 0.057078, 0.035860, 0.021978, 0.029111, + 0.029111, 0.024491, 0.029766, 0.014309, 0.021978, 0.009565, + 0.014645, 0.029536, 0.014536, 0.014536, 0.016784, 0.014559, + 0.014536, 0.019605, 0.014536, 0.014559, 0.014536, 0.031522, + 0.021076, 0.021006, 0.015044, 0.025640, 0.019573, 0.026288, + 0.024590, 0.037158, 0.071693, 0.100919, + ], + dtype=np.float32), + ), + "dolphins.csv" : + (cp.asarray(range(62), dtype=np.int32), + cp.asarray( + [0.01696534, 0.02465084, 0.01333804, 0.00962903, + 0.00507979, 0.01442816, 0.02005379, 0.01564308, + 0.01709825, 0.02345867, 0.01510835, 0.00507979, + 0.0048353 , 0.02615709, 0.03214436, 0.01988301, + 0.01662675, 0.03172837, 0.01939547, 0.01292825, + 0.02464085, 0.01693892, 0.00541593, 0.00986347, + 0.01690569, 0.01150429, 0.0112102 , 0.01713019, + 0.01484573, 0.02645844, 0.0153021 , 0.00541593, + 0.01330877, 0.02842296, 0.01591988, 0.00491821, + 0.02061337, 0.02987523, 0.02393915, 0.00776477, + 0.02196631, 0.01613769, 0.01761861, 0.02169104, + 0.01283079, 0.02951408, 0.00882587, 0.01733948, + 0.00526172, 0.00887672, 0.01923187, 0.03129924, + 0.01207255, 0.00818102, 0.02165103, 0.00749415, + 0.0083263 , 0.0300956 , 0.00496289, 0.01476788, + 0.00619018, 0.01103916, + ], + dtype=np.float32), + ), + "Simple_1" : + (cp.asarray(range(4), dtype=np.int32), + cp.asarray( + [0.11615585, 0.21488841, 0.2988108, 0.3701449], + dtype=np.float32) + ), + "Simple_2" : + (cp.asarray(range(6), dtype=np.int32), + cp.asarray( + [0.09902544, 0.17307726, 0.0732199, 0.1905103, + 0.12379099, 0.34037617, + ], + dtype=np.float32) + ), + } + # ============================================================================= # Pytest fixtures # ============================================================================= -# sg_graph_and_resource_handle fixture is defined in conftest.py. That fixture -# returns a preconstructed graph and corresponding resource handle for -# different datasets. -@pytest.fixture -def input_and_expected_output(sg_graph_objs): - (g, resource_handle, ds_name) = sg_graph_objs - return (g, resource_handle) +# fixtures used in this test module are defined in conftest.py + + +# ============================================================================= +# Helper functions +# ============================================================================= # ============================================================================= # Tests # ============================================================================= @pytest.mark.skip(reason="UNFINISHED") -def test_pagerank(input_and_expected_output): +def test_pagerank(sg_graph_objs): from pylibcugraph.experimental import pagerank - (g, resource_handle) = input_and_expected_output + (g, resource_handle, ds_name) = sg_graph_objs + expected_result = _test_data[ds_name] precomputed_vertex_out_weight_sums = None - alpha = 0.95 - epsilon = 0.0001 - max_iterations = 20 has_initial_guess = False do_expensive_check = False result = pagerank(resource_handle, g, precomputed_vertex_out_weight_sums, - alpha, - epsilon, - max_iterations, + _alpha, + _epsilon, + _max_iterations, has_initial_guess, do_expensive_check) - print(result) - raise NotImplementedError + + assert result == expected_result From dfee323c432974962390f8c17786edd38e092917 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Fri, 21 Jan 2022 22:28:58 -0600 Subject: [PATCH 19/43] flake8 fixes. --- .../pylibcugraph/tests/conftest.py | 2 +- .../pylibcugraph/tests/test_pagerank.py | 34 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/tests/conftest.py b/python/pylibcugraph/pylibcugraph/tests/conftest.py index 28d4845b101..35f59bec748 100644 --- a/python/pylibcugraph/pylibcugraph/tests/conftest.py +++ b/python/pylibcugraph/pylibcugraph/tests/conftest.py @@ -57,7 +57,7 @@ def __init__(self, srcs, dsts, weights, name): srcs=cp.asarray([0, 1, 1, 2, 2, 2, 3, 4], dtype=np.int32), dsts=cp.asarray([1, 3, 4, 0, 1, 3, 5, 5], dtype=np.int32), weights=cp.asarray([0.1, 2.1, 1.1, 5.1, 3.1, 4.1, 7.2, 3.2], - dtype=np.float32), + dtype=np.float32), name="Simple_2" ) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py index 776211cffae..34bd62f2859 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py @@ -25,7 +25,7 @@ # Map the names of input data to expected pagerank output # The result names correspond to the datasets defined in conftest.py -_test_data = {"karate.csv" : +_test_data = {"karate.csv": (cp.asarray(range(34), dtype=np.int32), cp.asarray( [0.096998, 0.052877, 0.057078, 0.035860, 0.021978, 0.029111, @@ -36,42 +36,42 @@ 0.024590, 0.037158, 0.071693, 0.100919, ], dtype=np.float32), - ), - "dolphins.csv" : + ), + "dolphins.csv": (cp.asarray(range(62), dtype=np.int32), cp.asarray( [0.01696534, 0.02465084, 0.01333804, 0.00962903, 0.00507979, 0.01442816, 0.02005379, 0.01564308, 0.01709825, 0.02345867, 0.01510835, 0.00507979, - 0.0048353 , 0.02615709, 0.03214436, 0.01988301, + 0.0048353, 0.02615709, 0.03214436, 0.01988301, 0.01662675, 0.03172837, 0.01939547, 0.01292825, 0.02464085, 0.01693892, 0.00541593, 0.00986347, - 0.01690569, 0.01150429, 0.0112102 , 0.01713019, - 0.01484573, 0.02645844, 0.0153021 , 0.00541593, + 0.01690569, 0.01150429, 0.0112102, 0.01713019, + 0.01484573, 0.02645844, 0.0153021, 0.00541593, 0.01330877, 0.02842296, 0.01591988, 0.00491821, 0.02061337, 0.02987523, 0.02393915, 0.00776477, 0.02196631, 0.01613769, 0.01761861, 0.02169104, 0.01283079, 0.02951408, 0.00882587, 0.01733948, 0.00526172, 0.00887672, 0.01923187, 0.03129924, 0.01207255, 0.00818102, 0.02165103, 0.00749415, - 0.0083263 , 0.0300956 , 0.00496289, 0.01476788, + 0.0083263, 0.0300956, 0.00496289, 0.01476788, 0.00619018, 0.01103916, ], dtype=np.float32), - ), - "Simple_1" : + ), + "Simple_1": (cp.asarray(range(4), dtype=np.int32), cp.asarray( [0.11615585, 0.21488841, 0.2988108, 0.3701449], dtype=np.float32) - ), - "Simple_2" : - (cp.asarray(range(6), dtype=np.int32), - cp.asarray( - [0.09902544, 0.17307726, 0.0732199, 0.1905103, - 0.12379099, 0.34037617, - ], - dtype=np.float32) + ), + "Simple_2": + (cp.asarray(range(6), dtype=np.int32), + cp.asarray( + [0.09902544, 0.17307726, 0.0732199, 0.1905103, + 0.12379099, 0.34037617, + ], + dtype=np.float32) ), } From 7257c05d2de06074be6b165d15506f2c7b968f27 Mon Sep 17 00:00:00 2001 From: Chuck Hastings Date: Sat, 22 Jan 2022 16:36:00 -0500 Subject: [PATCH 20/43] fix clang-format issues --- cpp/include/cugraph_c/algorithms.h | 15 ++++++++------- cpp/src/c_api/array.cpp | 6 +++--- cpp/src/c_api/array.hpp | 5 ++++- cpp/src/c_api/bfs.cpp | 16 ++++++++++------ cpp/src/c_api/extract_paths.cpp | 3 ++- cpp/src/c_api/graph_sg.cpp | 6 ++++-- cpp/src/c_api/pagerank.cpp | 27 +++++++++++++++------------ cpp/tests/c_api/sssp_test.c | 15 +++++++++------ 8 files changed, 55 insertions(+), 38 deletions(-) diff --git a/cpp/include/cugraph_c/algorithms.h b/cpp/include/cugraph_c/algorithms.h index d96b0e85cb8..fb958207a9d 100644 --- a/cpp/include/cugraph_c/algorithms.h +++ b/cpp/include/cugraph_c/algorithms.h @@ -280,13 +280,14 @@ typedef struct { * be populated if error code is not CUGRAPH_SUCCESS * @return error code */ -cugraph_error_code_t cugraph_extract_paths(const cugraph_resource_handle_t* handle, - cugraph_graph_t* graph, - const cugraph_type_erased_device_array_view_t* sources, - const cugraph_paths_result_t* paths_result, - const cugraph_type_erased_device_array_view_t* destinations, - cugraph_extract_paths_result_t** result, - cugraph_error_t** error); +cugraph_error_code_t cugraph_extract_paths( + const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + const cugraph_type_erased_device_array_view_t* sources, + const cugraph_paths_result_t* paths_result, + const cugraph_type_erased_device_array_view_t* destinations, + cugraph_extract_paths_result_t** result, + cugraph_error_t** error); /** * @brief Get the max path length from extract_paths result diff --git a/cpp/src/c_api/array.cpp b/cpp/src/c_api/array.cpp index 2ec6d1cb521..b5ac0285f8f 100644 --- a/cpp/src/c_api/array.cpp +++ b/cpp/src/c_api/array.cpp @@ -298,9 +298,9 @@ extern "C" cugraph_error_code_t cugraph_type_erased_device_array_view_copy( try { raft::handle_t const* raft_handle = reinterpret_cast(handle); auto internal_pointer_dst = - reinterpret_cast(dst); + reinterpret_cast(dst); auto internal_pointer_src = - reinterpret_cast(src); + reinterpret_cast(src); if (!raft_handle) { *error = reinterpret_cast( @@ -314,7 +314,7 @@ extern "C" cugraph_error_code_t cugraph_type_erased_device_array_view_copy( return CUGRAPH_INVALID_INPUT; } - raft::update_host(reinterpret_cast(internal_pointer_dst->data_), + raft::update_host(reinterpret_cast(internal_pointer_dst->data_), reinterpret_cast(internal_pointer_src->data_), internal_pointer_src->num_bytes(), raft_handle->get_stream()); diff --git a/cpp/src/c_api/array.hpp b/cpp/src/c_api/array.hpp index c0e9ccbb3d1..26465e05d3b 100644 --- a/cpp/src/c_api/array.hpp +++ b/cpp/src/c_api/array.hpp @@ -96,7 +96,10 @@ struct cugraph_type_erased_host_array_t { size_t num_bytes_; data_type_id_t type_; - auto view() { return new cugraph_type_erased_host_array_view_t{data_.get(), size_, num_bytes_, type_}; } + auto view() + { + return new cugraph_type_erased_host_array_view_t{data_.get(), size_, num_bytes_, type_}; + } }; } // namespace c_api diff --git a/cpp/src/c_api/bfs.cpp b/cpp/src/c_api/bfs.cpp index 467dae8784d..87e1051928d 100644 --- a/cpp/src/c_api/bfs.cpp +++ b/cpp/src/c_api/bfs.cpp @@ -145,21 +145,24 @@ extern "C" cugraph_type_erased_device_array_view_t* cugraph_paths_result_get_ver cugraph_paths_result_t* result) { auto internal_pointer = reinterpret_cast(result); - return reinterpret_cast(internal_pointer->vertex_ids_->view()); + return reinterpret_cast( + internal_pointer->vertex_ids_->view()); } extern "C" cugraph_type_erased_device_array_view_t* cugraph_paths_result_get_distances( cugraph_paths_result_t* result) { auto internal_pointer = reinterpret_cast(result); - return reinterpret_cast(internal_pointer->distances_->view()); + return reinterpret_cast( + internal_pointer->distances_->view()); } extern "C" cugraph_type_erased_device_array_view_t* cugraph_paths_result_get_predecessors( cugraph_paths_result_t* result) { auto internal_pointer = reinterpret_cast(result); - return reinterpret_cast(internal_pointer->predecessors_->view()); + return reinterpret_cast( + internal_pointer->predecessors_->view()); } extern "C" void cugraph_paths_result_free(cugraph_paths_result_t* result) @@ -185,9 +188,10 @@ extern "C" cugraph_error_code_t cugraph_bfs(const cugraph_resource_handle_t* han *error = nullptr; try { - auto p_handle = reinterpret_cast(handle); - auto p_graph = reinterpret_cast(graph); - auto p_sources = reinterpret_cast(sources); + auto p_handle = reinterpret_cast(handle); + auto p_graph = reinterpret_cast(graph); + auto p_sources = + reinterpret_cast(sources); cugraph::c_api::bfs_functor functor(*p_handle, p_graph, diff --git a/cpp/src/c_api/extract_paths.cpp b/cpp/src/c_api/extract_paths.cpp index ba2594c23a7..4772bc02958 100644 --- a/cpp/src/c_api/extract_paths.cpp +++ b/cpp/src/c_api/extract_paths.cpp @@ -181,7 +181,8 @@ extern "C" cugraph_error_code_t cugraph_extract_paths( auto p_paths_result = reinterpret_cast(paths_result); auto p_destinations = - reinterpret_cast(destinations); + reinterpret_cast( + destinations); cugraph::c_api::extract_paths_functor functor( *p_handle, p_graph, p_sources, p_paths_result, p_destinations); diff --git a/cpp/src/c_api/graph_sg.cpp b/cpp/src/c_api/graph_sg.cpp index f744ccbef18..61b14129dd0 100644 --- a/cpp/src/c_api/graph_sg.cpp +++ b/cpp/src/c_api/graph_sg.cpp @@ -189,8 +189,10 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( *error = nullptr; auto p_handle = reinterpret_cast(handle); - auto p_src = reinterpret_cast(src); - auto p_dst = reinterpret_cast(dst); + auto p_src = + reinterpret_cast(src); + auto p_dst = + reinterpret_cast(dst); auto p_weights = reinterpret_cast(weights); diff --git a/cpp/src/c_api/pagerank.cpp b/cpp/src/c_api/pagerank.cpp index 74c0309092d..ec2c1d4dd32 100644 --- a/cpp/src/c_api/pagerank.cpp +++ b/cpp/src/c_api/pagerank.cpp @@ -49,16 +49,17 @@ struct pagerank_functor : public abstract_functor { bool do_expensive_check_; cugraph_pagerank_result_t* result_{}; - pagerank_functor(raft::handle_t const& handle, - cugraph_graph_t* graph, - cugraph_type_erased_device_array_view_t const* precomputed_vertex_out_weight_sums, - cugraph_type_erased_device_array_view_t* personalization_vertices, - cugraph_type_erased_device_array_view_t const* personalization_values, - double alpha, - double epsilon, - size_t max_iterations, - bool has_initial_guess, - bool do_expensive_check) + pagerank_functor( + raft::handle_t const& handle, + cugraph_graph_t* graph, + cugraph_type_erased_device_array_view_t const* precomputed_vertex_out_weight_sums, + cugraph_type_erased_device_array_view_t* personalization_vertices, + cugraph_type_erased_device_array_view_t const* personalization_values, + double alpha, + double epsilon, + size_t max_iterations, + bool has_initial_guess, + bool do_expensive_check) : abstract_functor(), handle_(handle), graph_(graph), @@ -155,14 +156,16 @@ extern "C" cugraph_type_erased_device_array_view_t* cugraph_pagerank_result_get_ cugraph_pagerank_result_t* result) { auto internal_pointer = reinterpret_cast(result); - return reinterpret_cast(internal_pointer->vertex_ids_->view()); + return reinterpret_cast( + internal_pointer->vertex_ids_->view()); } extern "C" cugraph_type_erased_device_array_view_t* cugraph_pagerank_result_get_pageranks( cugraph_pagerank_result_t* result) { auto internal_pointer = reinterpret_cast(result); - return reinterpret_cast(internal_pointer->pageranks_->view()); + return reinterpret_cast( + internal_pointer->pageranks_->view()); } extern "C" void cugraph_pagerank_result_free(cugraph_pagerank_result_t* result) diff --git a/cpp/tests/c_api/sssp_test.c b/cpp/tests/c_api/sssp_test.c index 48ebb7c302b..4a906162ce0 100644 --- a/cpp/tests/c_api/sssp_test.c +++ b/cpp/tests/c_api/sssp_test.c @@ -58,9 +58,9 @@ int generic_sssp_test(vertex_t* h_src, p_handle, p_graph, source, cutoff, TRUE, FALSE, &p_result, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_sssp failed."); - cugraph_type_erased_device_array_t* vertices; - cugraph_type_erased_device_array_t* distances; - cugraph_type_erased_device_array_t* predecessors; + cugraph_type_erased_device_array_view_t* vertices; + cugraph_type_erased_device_array_view_t* distances; + cugraph_type_erased_device_array_view_t* predecessors; vertices = cugraph_paths_result_get_vertices(p_result); distances = cugraph_paths_result_get_distances(p_result); @@ -70,15 +70,15 @@ int generic_sssp_test(vertex_t* h_src, weight_t h_distances[num_vertices]; vertex_t h_predecessors[num_vertices]; - ret_code = cugraph_type_erased_device_array_copy_to_host( + ret_code = cugraph_type_erased_device_array_view_copy_to_host( p_handle, (byte_t*)h_vertices, vertices, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); - ret_code = cugraph_type_erased_device_array_copy_to_host( + ret_code = cugraph_type_erased_device_array_view_copy_to_host( p_handle, (byte_t*)h_distances, distances, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); - ret_code = cugraph_type_erased_device_array_copy_to_host( + ret_code = cugraph_type_erased_device_array_view_copy_to_host( p_handle, (byte_t*)h_predecessors, predecessors, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); @@ -92,6 +92,9 @@ int generic_sssp_test(vertex_t* h_src, "sssp predecessors don't match"); } + cugraph_type_erased_device_array_view_free(vertices); + cugraph_type_erased_device_array_view_free(distances); + cugraph_type_erased_device_array_view_free(predecessors); cugraph_paths_result_free(p_result); cugraph_sg_graph_free(p_graph); cugraph_free_resource_handle(p_handle); From ed722097aebc2337530f2fa09b51732f29d13bed Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Sat, 22 Jan 2022 16:53:21 -0600 Subject: [PATCH 21/43] More refactoring to make it clear which cython files are strictly for cugraph_c and which are additional specific to pylibcugraph, added bfs, sssp, and paths APIs to cugraph_c definitions, added placeholder test for SSSP. --- python/pylibcugraph/pylibcugraph/README.md | 0 python/pylibcugraph/pylibcugraph/__init__.py | 4 +- .../pylibcugraph/_cugraph_c/README.md | 0 .../pylibcugraph/_cugraph_c/_algorithms.pxd | 84 --------- .../pylibcugraph/_cugraph_c/algorithms.pxd | 163 ++++++++++++++++++ .../_cugraph_c/{_array.pxd => array.pxd} | 4 +- .../{_cugraph_api.pxd => cugraph_api.pxd} | 0 .../_cugraph_c/{_error.pxd => error.pxd} | 0 .../_cugraph_c/{_graph.pxd => graph.pxd} | 6 +- .../pylibcugraph/experimental/__init__.py | 33 +++- .../{_cugraph_c => }/graph_properties.pxd | 2 +- .../{_cugraph_c => }/graph_properties.pyx | 0 .../pylibcugraph/{_cugraph_c => }/graphs.pxd | 2 +- .../pylibcugraph/{_cugraph_c => }/graphs.pyx | 14 +- .../{_cugraph_c => }/pagerank.pyx | 16 +- .../{_cugraph_c => }/resource_handle.pxd | 2 +- .../{_cugraph_c => }/resource_handle.pyx | 2 +- .../pylibcugraph/tests/test_sssp.py | 62 +++++++ .../pylibcugraph/{_cugraph_c => }/utils.pxd | 2 +- .../pylibcugraph/{_cugraph_c => }/utils.pyx | 0 20 files changed, 281 insertions(+), 115 deletions(-) create mode 100644 python/pylibcugraph/pylibcugraph/README.md create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/README.md delete mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd create mode 100644 python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd rename python/pylibcugraph/pylibcugraph/_cugraph_c/{_array.pxd => array.pxd} (96%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{_cugraph_api.pxd => cugraph_api.pxd} (100%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{_error.pxd => error.pxd} (100%) rename python/pylibcugraph/pylibcugraph/_cugraph_c/{_graph.pxd => graph.pxd} (94%) rename python/pylibcugraph/pylibcugraph/{_cugraph_c => }/graph_properties.pxd (94%) rename python/pylibcugraph/pylibcugraph/{_cugraph_c => }/graph_properties.pyx (100%) rename python/pylibcugraph/pylibcugraph/{_cugraph_c => }/graphs.pxd (94%) rename python/pylibcugraph/pylibcugraph/{_cugraph_c => }/graphs.pyx (93%) rename python/pylibcugraph/pylibcugraph/{_cugraph_c => }/pagerank.pyx (89%) rename python/pylibcugraph/pylibcugraph/{_cugraph_c => }/resource_handle.pxd (93%) rename python/pylibcugraph/pylibcugraph/{_cugraph_c => }/resource_handle.pyx (95%) create mode 100644 python/pylibcugraph/pylibcugraph/tests/test_sssp.py rename python/pylibcugraph/pylibcugraph/{_cugraph_c => }/utils.pxd (94%) rename python/pylibcugraph/pylibcugraph/{_cugraph_c => }/utils.pyx (100%) diff --git a/python/pylibcugraph/pylibcugraph/README.md b/python/pylibcugraph/pylibcugraph/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/pylibcugraph/pylibcugraph/__init__.py b/python/pylibcugraph/pylibcugraph/__init__.py index cc76f196b2e..0385e2ef573 100644 --- a/python/pylibcugraph/pylibcugraph/__init__.py +++ b/python/pylibcugraph/pylibcugraph/__init__.py @@ -11,9 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph.components._connectivity import ( +from .components._connectivity import ( strongly_connected_components, weakly_connected_components, ) -from pylibcugraph import experimental +from . import experimental diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/README.md b/python/pylibcugraph/pylibcugraph/_cugraph_c/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd deleted file mode 100644 index 15c48ccc673..00000000000 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_algorithms.pxd +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pylibcugraph._cugraph_c._cugraph_api cimport ( - bool_t, - cugraph_resource_handle_t, -) -from pylibcugraph._cugraph_c._error cimport ( - cugraph_error_code_t, - cugraph_error_t, -) -from pylibcugraph._cugraph_c._array cimport ( - cugraph_type_erased_device_array_t, -) -from pylibcugraph._cugraph_c._graph cimport ( - cugraph_graph_t, -) - - -cdef extern from "cugraph_c/algorithms.h": - - ctypedef struct cugraph_pagerank_result_t: - pass - - # ctypedef struct cugraph_paths_result_t: - # pass - - # ctypedef struct cugraph_extract_paths_result_t: - # pass - - cdef cugraph_type_erased_device_array_t* \ - cugraph_pagerank_result_get_vertices( - cugraph_pagerank_result_t* result - ) - - cdef cugraph_type_erased_device_array_t* \ - cugraph_pagerank_result_get_pageranks( - cugraph_pagerank_result_t* result - ) - - cdef void \ - cugraph_pagerank_result_free( - cugraph_pagerank_result_t* result - ) - - cdef cugraph_error_code_t \ - cugraph_pagerank( - const cugraph_resource_handle_t* handle, - cugraph_graph_t* graph, - const cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, - double alpha, - double epsilon, - size_t max_iterations, - bool_t has_initial_guess, - bool_t do_expensive_check, - cugraph_pagerank_result_t** result, - cugraph_error_t** error - ) - - cdef cugraph_error_code_t \ - cugraph_personalized_pagerank( - const cugraph_resource_handle_t* handle, - cugraph_graph_t* graph, - const cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, - cugraph_type_erased_device_array_t* personalization_vertices, - const cugraph_type_erased_device_array_t* personalization_values, - double alpha, - double epsilon, - size_t max_iterations, - bool_t has_initial_guess, - bool_t do_expensive_check, - cugraph_pagerank_result_t** result, - cugraph_error_t** error - ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd new file mode 100644 index 00000000000..fa13f2ffe60 --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd @@ -0,0 +1,163 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph._cugraph_c.cugraph_api cimport ( + bool_t, + cugraph_resource_handle_t, +) +from pylibcugraph._cugraph_c.error cimport ( + cugraph_error_code_t, + cugraph_error_t, +) +from pylibcugraph._cugraph_c.array cimport ( + cugraph_type_erased_device_array_t, +) +from pylibcugraph._cugraph_c.graph cimport ( + cugraph_graph_t, +) + + +cdef extern from "cugraph_c/algorithms.h": + ########################################################################### + # pagerank + ctypedef struct cugraph_pagerank_result_t: + pass + + cdef cugraph_type_erased_device_array_t* \ + cugraph_pagerank_result_get_vertices( + cugraph_pagerank_result_t* result + ) + + cdef cugraph_type_erased_device_array_t* \ + cugraph_pagerank_result_get_pageranks( + cugraph_pagerank_result_t* result + ) + + cdef void \ + cugraph_pagerank_result_free( + cugraph_pagerank_result_t* result + ) + + cdef cugraph_error_code_t \ + cugraph_pagerank( + const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + const cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, + double alpha, + double epsilon, + size_t max_iterations, + bool_t has_initial_guess, + bool_t do_expensive_check, + cugraph_pagerank_result_t** result, + cugraph_error_t** error + ) + + cdef cugraph_error_code_t \ + cugraph_personalized_pagerank( + const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + const cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, + cugraph_type_erased_device_array_t* personalization_vertices, + const cugraph_type_erased_device_array_t* personalization_values, + double alpha, + double epsilon, + size_t max_iterations, + bool_t has_initial_guess, + bool_t do_expensive_check, + cugraph_pagerank_result_t** result, + cugraph_error_t** error + ) + + ########################################################################### + # paths and path extraction + ctypedef struct cugraph_paths_result_t: + pass + + cdef cugraph_type_erased_device_array_t* \ + cugraph_paths_result_get_vertices( + cugraph_paths_result_t* result + ) + + cdef cugraph_type_erased_device_array_t* \ + cugraph_paths_result_get_distances( + cugraph_paths_result_t* result + ) + + cdef cugraph_type_erased_device_array_t* \ + cugraph_paths_result_get_predecessors( + cugraph_paths_result_t* result + ) + + cdef void \ + cugraph_paths_result_free( + cugraph_paths_result_t* result + ) + + ctypedef struct cugraph_extract_paths_result_t: + pass + + cdef cugraph_error_code_t \ + cugraph_extract_paths( + const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + const cugraph_type_erased_device_array_t* sources, + const cugraph_paths_result_t* paths_result, + const cugraph_type_erased_device_array_t* destinations, + cugraph_extract_paths_result_t** result, + cugraph_error_t** error + ) + + cdef size_t \ + cugraph_extract_paths_result_get_max_path_length( + cugraph_extract_paths_result_t* result + ) + + cdef cugraph_type_erased_device_array_t* \ + cugraph_extract_paths_result_get_paths( + cugraph_extract_paths_result_t* result + ) + + cdef void \ + cugraph_extract_paths_result_free( + cugraph_extract_paths_result_t* result + ) + + ########################################################################### + # bfs + cdef cugraph_error_code_t \ + cugraph_bfs( + const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + # FIXME: this may become const + cugraph_type_erased_device_array_t* sources, + bool_t direction_optimizing, + size_t depth_limit, + bool_t compute_predecessors, + bool_t do_expensive_check, + cugraph_paths_result_t** result, + cugraph_error_t** error + ) + + ########################################################################### + # sssp + cdef cugraph_error_code_t \ + cugraph_sssp( + const cugraph_resource_handle_t* handle, + cugraph_graph_t* graph, + size_t source, + double cutoff, + bool_t compute_predecessors, + bool_t do_expensive_check, + cugraph_paths_result_t** result, + cugraph_error_t** error + ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd similarity index 96% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd index 0366a94655e..6239f9f174c 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_array.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd @@ -11,11 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c._error cimport ( +from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, cugraph_error_t, ) -from pylibcugraph._cugraph_c._cugraph_api cimport ( +from pylibcugraph._cugraph_c.cugraph_api cimport ( cugraph_resource_handle_t, data_type_id_t, byte_t, diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd similarity index 100% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/_cugraph_api.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/error.pxd similarity index 100% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/_error.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/error.pxd diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd similarity index 94% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd rename to python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd index 3dc3bfdc0d9..97af3ba93d2 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/_graph.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd @@ -11,15 +11,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c._cugraph_api cimport ( +from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, cugraph_resource_handle_t, ) -from pylibcugraph._cugraph_c._error cimport ( +from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, cugraph_error_t, ) -from pylibcugraph._cugraph_c._array cimport ( +from pylibcugraph._cugraph_c.array cimport ( cugraph_type_erased_device_array_t, cugraph_type_erased_host_array_t, ) diff --git a/python/pylibcugraph/pylibcugraph/experimental/__init__.py b/python/pylibcugraph/pylibcugraph/experimental/__init__.py index d0c65823e89..3a94036d60d 100644 --- a/python/pylibcugraph/pylibcugraph/experimental/__init__.py +++ b/python/pylibcugraph/pylibcugraph/experimental/__init__.py @@ -11,16 +11,41 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +The "experimental" package contains packages, functions, classes, etc. that +are ready for use but do not have their API signatures or implementation +finalized yet. This allows users to provide early feedback while still +permitting bigger design changes to take place. + +ALL APIS IN EXPERIMENTAL ARE SUBJECT TO CHANGE OR REMOVAL. + +Calling experimental objects will raise a PendingDeprecationWarning warning. + +If an object is "promoted" to the public API, the experimental namespace will +continue to also have that object present for at least another release. A +different warning will be output in that case, indicating that the experimental +API has been promoted and will no longer be importable from experimental much +longer. +""" + from pylibcugraph.utilities.api_tools import experimental_warning_wrapper -from pylibcugraph._cugraph_c.graphs import EXPERIMENTAL__SGGraph +# experimental_warning_wrapper() wraps the object in a function that provides +# the appropriate warning about using experimental code. + +# The convention of naming functions with the "EXPERIMENTAL__" prefix +# discourages users from directly importing experimental objects that don't have +# the appropriate warnings, such as what the wrapper and the "experimental" +# namespace name provides. + +from pylibcugraph.graphs import EXPERIMENTAL__SGGraph SGGraph = experimental_warning_wrapper(EXPERIMENTAL__SGGraph) -from pylibcugraph._cugraph_c.resource_handle import EXPERIMENTAL__ResourceHandle +from pylibcugraph.resource_handle import EXPERIMENTAL__ResourceHandle ResourceHandle = experimental_warning_wrapper(EXPERIMENTAL__ResourceHandle) -from pylibcugraph._cugraph_c.graph_properties import EXPERIMENTAL__GraphProperties +from pylibcugraph.graph_properties import EXPERIMENTAL__GraphProperties GraphProperties = experimental_warning_wrapper(EXPERIMENTAL__GraphProperties) -from pylibcugraph._cugraph_c.pagerank import EXPERIMENTAL__pagerank +from pylibcugraph.pagerank import EXPERIMENTAL__pagerank pagerank = experimental_warning_wrapper(EXPERIMENTAL__pagerank) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pxd b/python/pylibcugraph/pylibcugraph/graph_properties.pxd similarity index 94% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pxd rename to python/pylibcugraph/pylibcugraph/graph_properties.pxd index d2cedcb3316..37511503360 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pxd +++ b/python/pylibcugraph/pylibcugraph/graph_properties.pxd @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c._graph cimport ( +from pylibcugraph._cugraph_c.graph cimport ( cugraph_graph_properties_t, ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pyx b/python/pylibcugraph/pylibcugraph/graph_properties.pyx similarity index 100% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/graph_properties.pyx rename to python/pylibcugraph/pylibcugraph/graph_properties.pyx diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pxd b/python/pylibcugraph/pylibcugraph/graphs.pxd similarity index 94% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pxd rename to python/pylibcugraph/pylibcugraph/graphs.pxd index dfb5be5426b..0c6bc32d52b 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pxd +++ b/python/pylibcugraph/pylibcugraph/graphs.pxd @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c._graph cimport ( +from pylibcugraph._cugraph_c.graph cimport ( cugraph_graph_t, ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx b/python/pylibcugraph/pylibcugraph/graphs.pyx similarity index 93% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx rename to python/pylibcugraph/pylibcugraph/graphs.pyx index 98eb10c4109..08f33276a9c 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graphs.pyx +++ b/python/pylibcugraph/pylibcugraph/graphs.pyx @@ -11,33 +11,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c._cugraph_api cimport ( +from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, cugraph_resource_handle_t, data_type_id_t, ) -from pylibcugraph._cugraph_c._error cimport ( +from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, cugraph_error_t, ) -from pylibcugraph._cugraph_c._array cimport ( +from pylibcugraph._cugraph_c.array cimport ( cugraph_type_erased_device_array_t, cugraph_type_erased_device_array_create, cugraph_type_erased_device_array_free, ) -from pylibcugraph._cugraph_c._graph cimport ( +from pylibcugraph._cugraph_c.graph cimport ( cugraph_graph_t, cugraph_sg_graph_create, cugraph_graph_properties_t, cugraph_sg_graph_free, ) -from pylibcugraph._cugraph_c.resource_handle cimport ( +from pylibcugraph.resource_handle cimport ( EXPERIMENTAL__ResourceHandle, ) -from pylibcugraph._cugraph_c.graph_properties cimport ( +from pylibcugraph.graph_properties cimport ( EXPERIMENTAL__GraphProperties, ) -from pylibcugraph._cugraph_c.utils cimport ( +from pylibcugraph.utils cimport ( assert_success, ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx b/python/pylibcugraph/pylibcugraph/pagerank.pyx similarity index 89% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx rename to python/pylibcugraph/pylibcugraph/pagerank.pyx index fa1381a4455..90a13480a97 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/pagerank.pyx @@ -11,35 +11,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c._cugraph_api cimport ( +from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, #data_type_id_t, cugraph_resource_handle_t, ) -from pylibcugraph._cugraph_c._error cimport ( +from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, cugraph_error_t, ) -from pylibcugraph._cugraph_c._array cimport ( +from pylibcugraph._cugraph_c.array cimport ( cugraph_type_erased_device_array_t, cugraph_type_erased_device_array_create, cugraph_type_erased_device_array_free, ) -from pylibcugraph._cugraph_c._graph cimport ( +from pylibcugraph._cugraph_c.graph cimport ( cugraph_graph_t, ) -from pylibcugraph._cugraph_c._algorithms cimport ( +from pylibcugraph._cugraph_c.algorithms cimport ( cugraph_pagerank_result_t, cugraph_pagerank, ) -from pylibcugraph._cugraph_c.resource_handle cimport ( +from pylibcugraph.resource_handle cimport ( EXPERIMENTAL__ResourceHandle, ) -from pylibcugraph._cugraph_c.graphs cimport ( +from pylibcugraph.graphs cimport ( EXPERIMENTAL__Graph, ) -from pylibcugraph._cugraph_c.utils cimport ( +from pylibcugraph.utils cimport ( assert_success, ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pxd b/python/pylibcugraph/pylibcugraph/resource_handle.pxd similarity index 93% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pxd rename to python/pylibcugraph/pylibcugraph/resource_handle.pxd index 5bf16ae43c9..afe6ba18a27 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pxd +++ b/python/pylibcugraph/pylibcugraph/resource_handle.pxd @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c._cugraph_api cimport ( +from pylibcugraph._cugraph_c.cugraph_api cimport ( cugraph_resource_handle_t, ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx b/python/pylibcugraph/pylibcugraph/resource_handle.pyx similarity index 95% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx rename to python/pylibcugraph/pylibcugraph/resource_handle.pyx index 1a008166793..30fc62ffd6d 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/resource_handle.pyx +++ b/python/pylibcugraph/pylibcugraph/resource_handle.pyx @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c._cugraph_api cimport ( +from pylibcugraph._cugraph_c.cugraph_api cimport ( cugraph_create_resource_handle, cugraph_free_resource_handle, ) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_sssp.py b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py new file mode 100644 index 00000000000..d99ed3abffb --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py @@ -0,0 +1,62 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +# import cupy as cp +# import numpy as np + + +# ============================================================================= +# Test data +# ============================================================================= + +# Map the names of input data to expected pagerank output +# The result names correspond to the datasets defined in conftest.py +_test_data = {"karate.csv": + None, + "dolphins.csv": + None, + "Simple_1": + None, + "Simple_2": + None, + } + +# ============================================================================= +# Pytest fixtures +# ============================================================================= +# fixtures used in this test module are defined in conftest.py + + +# ============================================================================= +# Helper functions +# ============================================================================= + + +# ============================================================================= +# Tests +# ============================================================================= +@pytest.mark.skip(reason="UNFINISHED") +def test_sssp(sg_graph_objs): + from pylibcugraph.experimental import sssp + + (g, resource_handle, ds_name) = sg_graph_objs + expected_result = _test_data[ds_name] + + do_expensive_check = False + + result = sssp(resource_handle, + g, + do_expensive_check) + + assert result == expected_result diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pxd b/python/pylibcugraph/pylibcugraph/utils.pxd similarity index 94% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pxd rename to python/pylibcugraph/pylibcugraph/utils.pxd index 5965cc66364..1836196124b 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pxd +++ b/python/pylibcugraph/pylibcugraph/utils.pxd @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pylibcugraph._cugraph_c._error cimport ( +from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, cugraph_error_t, ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pyx b/python/pylibcugraph/pylibcugraph/utils.pyx similarity index 100% rename from python/pylibcugraph/pylibcugraph/_cugraph_c/utils.pyx rename to python/pylibcugraph/pylibcugraph/utils.pyx From b7131ad52bd759c4e549f78d89310adf3556e336 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Sat, 22 Jan 2022 19:04:46 -0600 Subject: [PATCH 22/43] Updated sssp placeholder test to have expected results, added initial cython binding for sssp. --- .../pylibcugraph/experimental/__init__.py | 3 + python/pylibcugraph/pylibcugraph/pagerank.pyx | 1 - python/pylibcugraph/pylibcugraph/sssp.pyx | 83 +++++++++++++++++ .../pylibcugraph/tests/test_sssp.py | 89 ++++++++++++++++--- 4 files changed, 163 insertions(+), 13 deletions(-) create mode 100644 python/pylibcugraph/pylibcugraph/sssp.pyx diff --git a/python/pylibcugraph/pylibcugraph/experimental/__init__.py b/python/pylibcugraph/pylibcugraph/experimental/__init__.py index 3a94036d60d..81d95cd56c5 100644 --- a/python/pylibcugraph/pylibcugraph/experimental/__init__.py +++ b/python/pylibcugraph/pylibcugraph/experimental/__init__.py @@ -49,3 +49,6 @@ from pylibcugraph.pagerank import EXPERIMENTAL__pagerank pagerank = experimental_warning_wrapper(EXPERIMENTAL__pagerank) + +from pylibcugraph.sssp import EXPERIMENTAL__sssp +sssp = experimental_warning_wrapper(EXPERIMENTAL__sssp) diff --git a/python/pylibcugraph/pylibcugraph/pagerank.pyx b/python/pylibcugraph/pylibcugraph/pagerank.pyx index 90a13480a97..e26bd7e412d 100644 --- a/python/pylibcugraph/pylibcugraph/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/pagerank.pyx @@ -44,7 +44,6 @@ from pylibcugraph.utils cimport ( ) - def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, EXPERIMENTAL__Graph graph, precomputed_vertex_out_weight_sums, diff --git a/python/pylibcugraph/pylibcugraph/sssp.pyx b/python/pylibcugraph/pylibcugraph/sssp.pyx new file mode 100644 index 00000000000..43e5d955e90 --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/sssp.pyx @@ -0,0 +1,83 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pylibcugraph._cugraph_c.cugraph_api cimport ( + bool_t, + #data_type_id_t, + cugraph_resource_handle_t, +) +from pylibcugraph._cugraph_c.error cimport ( + cugraph_error_code_t, + cugraph_error_t, +) +from pylibcugraph._cugraph_c.graph cimport ( + cugraph_graph_t, +) +from pylibcugraph._cugraph_c.algorithms cimport ( + cugraph_paths_result_t, + cugraph_sssp, +) + +from pylibcugraph.resource_handle cimport ( + EXPERIMENTAL__ResourceHandle, +) +from pylibcugraph.graphs cimport ( + EXPERIMENTAL__Graph, +) +from pylibcugraph.utils cimport ( + assert_success, +) + + +def EXPERIMENTAL__sssp(EXPERIMENTAL__ResourceHandle resource_handle, + EXPERIMENTAL__Graph graph, + size_t source, + double cutoff, + bool_t compute_predecessors, + bool_t do_expensive_check): + """ + """ + cdef cugraph_resource_handle_t* c_resource_handle_ptr = resource_handle.c_resource_handle_ptr + cdef cugraph_graph_t* c_graph_ptr = graph.c_graph_ptr + + cdef cugraph_paths_result_t* result_ptr + cdef cugraph_error_code_t error_code + cdef cugraph_error_t* error_ptr + + error_code = cugraph_sssp(c_resource_handle_ptr, + c_graph_ptr, + source, + cutoff, + compute_predecessors, + do_expensive_check, + &result_ptr, + &error_ptr) + + +""" + cdef cugraph_pagerank_result_t* my_result + cugraph_pagerank(... , &my_result, ...) + + cdef cugraph_type_erased_device_array_t* verts + cdef cugraph_type_erased_device_array_t* prs + verts = cugraph_pagerank_result_get_vertices(my_result) + prs = cugraph_pagerank_result_get_pageranks(my_result) + + do device-device copy on verts to user array + do device-device copy on prs to user array +/* +size_t cugraph_type_erased_device_array_size(const cugraph_type_erased_device_array_t* p); +data_type_id_t cugraph_type_erased_device_array_type(const cugraph_type_erased_device_array_t* p); +const void* cugraph_type_erased_device_array_pointer(const cugraph_type_erased_device_array_t* p); +*/ +""" diff --git a/python/pylibcugraph/pylibcugraph/tests/test_sssp.py b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py index d99ed3abffb..fad071f6f41 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_sssp.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py @@ -12,8 +12,8 @@ # limitations under the License. import pytest -# import cupy as cp -# import numpy as np +import cupy as cp +import numpy as np # ============================================================================= @@ -22,14 +22,68 @@ # Map the names of input data to expected pagerank output # The result names correspond to the datasets defined in conftest.py -_test_data = {"karate.csv": - None, - "dolphins.csv": - None, - "Simple_1": - None, - "Simple_2": - None, +_test_data = {"karate.csv": { + "start_vertex": 1, + "vertex": cp.asarray(range(34), dtype=np.int32), + "distance": cp.asarray( + [1., 0., 1., 1., 2., 2., 2., 1., 2., 2., 2., 2., 2., 1., + 3., 3., 3., 1., 3., 1., 3., 1., 3., 3., 3., 3., 3., 2., + 2., 3., 1., 2., 2., 2., + ], + dtype=np.float32), + "predecessor": cp.asarray( + [1, -1, 1, 1, 0, 0, 0, 1, 0, 2, 0, 0, 0, 1, 32, + 32, 5, 1, 32, 1, 32, 1, 32, 32, 27, 31, 33, 2, 2, 32, + 1, 0, 2, 13, + ], + dtype=np.int32), + }, + "dolphins.csv": { + "start_vertex": 1, + "vertex": cp.asarray(range(62), dtype=np.int32), + "distance": cp.asarray( + [3., 0., 4., 3., 4., 3., 2., 2., 2., 2., 3., 4., 4., 2., + 3., 3., 3., 1., 3., 1., 2., 3., 2., 2., 4., 2., 1., 1., + 1., 4., 2., 2., 3., 3., 3., 5., 1., 2., 3., 2., 2., 1., + 3., 3., 3., 3., 4., 2., 3., 4., 3., 3., 3., 4., 1., 4., + 3., 2., 4., 2., 4., 3., + ], + dtype=np.float32), + "predecessor": cp.asarray( + [40, -1, 10, 59, 51, 13, 54, 54, 28, 41, 47, 51, 33, 41, + 37, 40, 37, 1, 20, 1, 28, 37, 17, 36, 45, 17, 1, 1, + 1, 10, 19, 17, 9, 37, 37, 29, 1, 36, 20, 36, 36, 1, + 30, 37, 20, 23, 43, 28, 57, 34, 20, 23, 40, 43, 1, 51, + 6, 41, 38, 36, 32, 37, + ], + dtype=np.int32), + }, + "Simple_1": { + "start_vertex": 1, + "vertex": cp.asarray(range(34), dtype=np.int32), + "distance": cp.asarray( + [3.4028235e+38, 0.0000000e+00, 1.0000000e+00, + 2.0000000e+00, + ], + dtype=np.float32), + "predecessor": cp.asarray( + [-1, -1, 1, 2, + ], + dtype=np.int32), + }, + "Simple_2": { + "start_vertex": 1, + "vertex": cp.asarray(range(34), dtype=np.int32), + "distance": cp.asarray( + [3.4028235e+38, 0.0000000e+00, 3.4028235e+38, + 2.0999999e+00, 1.1000000e+00, 4.3000002e+00 + ], + dtype=np.float32), + "predecessor": cp.asarray( + [-1, -1, -1, 1, 1, 4, + ], + dtype=np.int32), + }, } # ============================================================================= @@ -51,12 +105,23 @@ def test_sssp(sg_graph_objs): from pylibcugraph.experimental import sssp (g, resource_handle, ds_name) = sg_graph_objs - expected_result = _test_data[ds_name] + (source, + expected_vertices, + expected_distances, + expected_predecessors) = _test_data[ds_name].values() + + cutoff = 999999999 # maximum edge weight sum to consider + compute_predecessors = True do_expensive_check = False result = sssp(resource_handle, g, + source, + cutoff, + compute_predecessors, do_expensive_check) - assert result == expected_result + assert result == (expected_vertices, + expected_distances, + expected_predecessors) From 786a54ada55f0d9690a4c6573be673969fb69166 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Sat, 22 Jan 2022 21:55:38 -0600 Subject: [PATCH 23/43] Refactored to add a util function for checking input types, started adding code to process pagerank return values. --- python/pylibcugraph/pylibcugraph/graphs.pyx | 13 +++------ python/pylibcugraph/pylibcugraph/pagerank.pyx | 29 +++++++++++++++++-- python/pylibcugraph/pylibcugraph/utils.pxd | 2 ++ python/pylibcugraph/pylibcugraph/utils.pyx | 12 ++++++++ 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/graphs.pyx b/python/pylibcugraph/pylibcugraph/graphs.pyx index 08f33276a9c..e56a90cccff 100644 --- a/python/pylibcugraph/pylibcugraph/graphs.pyx +++ b/python/pylibcugraph/pylibcugraph/graphs.pyx @@ -39,6 +39,7 @@ from pylibcugraph.graph_properties cimport ( ) from pylibcugraph.utils cimport ( assert_success, + assert_CAI_type, ) @@ -67,15 +68,9 @@ cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): if not(isinstance(expensive_check, (int, bool))): raise TypeError("expected int or bool for expensive_check, got " f"{type(expensive_check)}") - if not(hasattr(src_array, "__cuda_array_interface__")): - raise TypeError("src_array does not have required " - "__cuda_array_interface__ attr") - if not(hasattr(dst_array, "__cuda_array_interface__")): - raise TypeError("dst_array does not have required " - "__cuda_array_interface__ attr") - if not(hasattr(weight_array, "__cuda_array_interface__")): - raise TypeError("weight_array does not have required " - "__cuda_array_interface__ attr") + assert_CAI_type(src_array, "src_array") + assert_CAI_type(dst_array, "dst_array") + assert_CAI_type(weight_array, "weight_array") cdef cugraph_error_t* error_ptr cdef cugraph_error_code_t error_code diff --git a/python/pylibcugraph/pylibcugraph/pagerank.pyx b/python/pylibcugraph/pylibcugraph/pagerank.pyx index e26bd7e412d..b700e8729e1 100644 --- a/python/pylibcugraph/pylibcugraph/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/pagerank.pyx @@ -13,7 +13,7 @@ from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, - #data_type_id_t, + data_type_id_t, cugraph_resource_handle_t, ) from pylibcugraph._cugraph_c.error cimport ( @@ -22,6 +22,8 @@ from pylibcugraph._cugraph_c.error cimport ( ) from pylibcugraph._cugraph_c.array cimport ( cugraph_type_erased_device_array_t, + cugraph_type_erased_device_array_size, + cugraph_type_erased_device_array_type, cugraph_type_erased_device_array_create, cugraph_type_erased_device_array_free, ) @@ -31,6 +33,8 @@ from pylibcugraph._cugraph_c.graph cimport ( from pylibcugraph._cugraph_c.algorithms cimport ( cugraph_pagerank_result_t, cugraph_pagerank, + cugraph_pagerank_result_get_vertices, + cugraph_pagerank_result_get_pageranks, ) from pylibcugraph.resource_handle cimport ( @@ -41,6 +45,7 @@ from pylibcugraph.graphs cimport ( ) from pylibcugraph.utils cimport ( assert_success, + assert_CAI_type, ) @@ -54,10 +59,17 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, bool_t do_expensive_check): """ """ + assert_CAI_type(precomputed_vertex_out_weight_sums, + "precomputed_vertex_out_weight_sums", + allow_None=True) + cdef cugraph_resource_handle_t* c_resource_handle_ptr = resource_handle.c_resource_handle_ptr cdef cugraph_graph_t* c_graph_ptr = graph.c_graph_ptr - #cdef cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums_ptr = precomputed_vertex_out_weight_sums cdef cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums_ptr = NULL + if precomputed_vertex_out_weight_sums: + raise NotImplementedError("None is the only supported value for precomputed_vertex_out_weight_sums") + #precomputed_vertex_out_weight_sums_ptr = precomputed_vertex_out_weight_sums + pass cdef cugraph_pagerank_result_t* result_ptr cdef cugraph_error_code_t error_code @@ -73,6 +85,19 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, do_expensive_check, &result_ptr, &error_ptr) + assert_success(error_code, error_ptr, "cugraph_pagerank") + + cdef cugraph_type_erased_device_array_t* vertices_ptr + cdef cugraph_type_erased_device_array_t* pageranks_ptr + vertices_ptr = cugraph_pagerank_result_get_vertices(result_ptr) + pageranks_ptr = cugraph_pagerank_result_get_pageranks(result_ptr) + + cdef size_t num_verts + num_verts = cugraph_type_erased_device_array_size(vertices_ptr) + cdef data_type_id_t verts_type + verts_type = cugraph_type_erased_device_array_type(vertices_ptr) + cdef data_type_id_t pageranks_type + pageranks_type = cugraph_type_erased_device_array_type(pageranks_ptr) """ diff --git a/python/pylibcugraph/pylibcugraph/utils.pxd b/python/pylibcugraph/pylibcugraph/utils.pxd index 1836196124b..c2570156aeb 100644 --- a/python/pylibcugraph/pylibcugraph/utils.pxd +++ b/python/pylibcugraph/pylibcugraph/utils.pxd @@ -20,3 +20,5 @@ from pylibcugraph._cugraph_c.error cimport ( cdef assert_success(cugraph_error_code_t code, cugraph_error_t* err, api_name) + +cdef assert_CAI_type(obj, var_name, allow_None=*) diff --git a/python/pylibcugraph/pylibcugraph/utils.pyx b/python/pylibcugraph/pylibcugraph/utils.pyx index b3bbbf7ef71..5d4c38294e1 100644 --- a/python/pylibcugraph/pylibcugraph/utils.pyx +++ b/python/pylibcugraph/pylibcugraph/utils.pyx @@ -19,3 +19,15 @@ cdef assert_success(cugraph_error_code_t code, if code != cugraph_error_code_t.CUGRAPH_SUCCESS: # FIXME: extract message using cugraph_error_message() raise RuntimeError(f"non-success value returned from {api_name}") + + +cdef assert_CAI_type(obj, var_name, allow_None=False): + if allow_None: + if obj is None: + return + msg = f"{var_name} must be None or support __cuda_array_interface__" + else: + msg = f"{var_name} does not support __cuda_array_interface__" + + if not(hasattr(obj, "__cuda_array_interface__")): + raise TypeError(msg) From 41d52706e14815da700412a28e18be23452dd337 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Sun, 23 Jan 2022 12:37:53 -0600 Subject: [PATCH 24/43] More work converting return types, added cython directive to use python 3 syntax. --- .../pylibcugraph/_cugraph_c/algorithms.pxd | 3 ++ .../pylibcugraph/_cugraph_c/array.pxd | 3 ++ .../pylibcugraph/_cugraph_c/cugraph_api.pxd | 3 ++ .../pylibcugraph/_cugraph_c/error.pxd | 2 + .../pylibcugraph/_cugraph_c/graph.pxd | 3 ++ .../pylibcugraph/graph_properties.pxd | 3 ++ .../pylibcugraph/graph_properties.pyx | 2 + python/pylibcugraph/pylibcugraph/graphs.pxd | 3 ++ python/pylibcugraph/pylibcugraph/graphs.pyx | 3 ++ python/pylibcugraph/pylibcugraph/pagerank.pyx | 37 +++++++++++++++---- .../pylibcugraph/resource_handle.pxd | 3 ++ .../pylibcugraph/resource_handle.pyx | 3 ++ python/pylibcugraph/pylibcugraph/sssp.pyx | 22 ++--------- python/pylibcugraph/pylibcugraph/utils.pxd | 8 ++++ python/pylibcugraph/pylibcugraph/utils.pyx | 18 +++++++++ 15 files changed, 91 insertions(+), 25 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd index fa13f2ffe60..7d66caac39c 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd @@ -11,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 + from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, cugraph_resource_handle_t, diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd index 6239f9f174c..711576c1366 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd @@ -11,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 + from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, cugraph_error_t, diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd index 7b4299e550d..0d3b0fc0185 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd @@ -11,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 + from libc.stdint cimport int8_t diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/error.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/error.pxd index 1610deb41cd..4b20daa1135 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/error.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/error.pxd @@ -11,6 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 cdef extern from "cugraph_c/error.h": diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd index 97af3ba93d2..c46e186b3cd 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd @@ -11,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 + from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, cugraph_resource_handle_t, diff --git a/python/pylibcugraph/pylibcugraph/graph_properties.pxd b/python/pylibcugraph/pylibcugraph/graph_properties.pxd index 37511503360..ed3290186b2 100644 --- a/python/pylibcugraph/pylibcugraph/graph_properties.pxd +++ b/python/pylibcugraph/pylibcugraph/graph_properties.pxd @@ -11,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 + from pylibcugraph._cugraph_c.graph cimport ( cugraph_graph_properties_t, ) diff --git a/python/pylibcugraph/pylibcugraph/graph_properties.pyx b/python/pylibcugraph/pylibcugraph/graph_properties.pyx index d178e4c7318..dc8b2a51225 100644 --- a/python/pylibcugraph/pylibcugraph/graph_properties.pyx +++ b/python/pylibcugraph/pylibcugraph/graph_properties.pyx @@ -11,6 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 cdef class EXPERIMENTAL__GraphProperties: """ diff --git a/python/pylibcugraph/pylibcugraph/graphs.pxd b/python/pylibcugraph/pylibcugraph/graphs.pxd index 0c6bc32d52b..9da256f9928 100644 --- a/python/pylibcugraph/pylibcugraph/graphs.pxd +++ b/python/pylibcugraph/pylibcugraph/graphs.pxd @@ -11,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 + from pylibcugraph._cugraph_c.graph cimport ( cugraph_graph_t, ) diff --git a/python/pylibcugraph/pylibcugraph/graphs.pyx b/python/pylibcugraph/pylibcugraph/graphs.pyx index e56a90cccff..5274cdcdeec 100644 --- a/python/pylibcugraph/pylibcugraph/graphs.pyx +++ b/python/pylibcugraph/pylibcugraph/graphs.pyx @@ -11,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 + from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, cugraph_resource_handle_t, diff --git a/python/pylibcugraph/pylibcugraph/pagerank.pyx b/python/pylibcugraph/pylibcugraph/pagerank.pyx index b700e8729e1..94035b2205f 100644 --- a/python/pylibcugraph/pylibcugraph/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/pagerank.pyx @@ -11,6 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 + +from libc.stdint cimport uintptr_t + from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, data_type_id_t, @@ -46,8 +51,12 @@ from pylibcugraph.graphs cimport ( from pylibcugraph.utils cimport ( assert_success, assert_CAI_type, + get_numpy_type_from_c_type, ) +import cupy +import numpy + def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, EXPERIMENTAL__Graph graph, @@ -67,7 +76,7 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, cdef cugraph_graph_t* c_graph_ptr = graph.c_graph_ptr cdef cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums_ptr = NULL if precomputed_vertex_out_weight_sums: - raise NotImplementedError("None is the only supported value for precomputed_vertex_out_weight_sums") + raise NotImplementedError("None is temporarily the only supported value for precomputed_vertex_out_weight_sums") #precomputed_vertex_out_weight_sums_ptr = precomputed_vertex_out_weight_sums pass @@ -87,18 +96,32 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, &error_ptr) assert_success(error_code, error_ptr, "cugraph_pagerank") + # Extract individual device array pointers from result cdef cugraph_type_erased_device_array_t* vertices_ptr cdef cugraph_type_erased_device_array_t* pageranks_ptr vertices_ptr = cugraph_pagerank_result_get_vertices(result_ptr) pageranks_ptr = cugraph_pagerank_result_get_pageranks(result_ptr) - cdef size_t num_verts - num_verts = cugraph_type_erased_device_array_size(vertices_ptr) - cdef data_type_id_t verts_type - verts_type = cugraph_type_erased_device_array_type(vertices_ptr) - cdef data_type_id_t pageranks_type - pageranks_type = cugraph_type_erased_device_array_type(pageranks_ptr) + # Extract meta-data needed to copy results + vertex_numpy_type = get_numpy_type_from_c_type( + cugraph_type_erased_device_array_type(vertices_ptr)) + pagerank_numpy_type = get_numpy_type_from_c_type( + cugraph_type_erased_device_array_type(pageranks_ptr)) + num_vertices = cugraph_type_erased_device_array_size(vertices_ptr) + + # Set up cupy arrays to return and copy results to those + cupy_vertices = cupy.array(numpy.zeros(num_vertices), + dtype=vertex_numpy_type) + cupy_pageranks = cupy.array(numpy.zeros(num_vertices), + dtype=pagerank_numpy_type) + + cdef uintptr_t cupy_vertices_ptr = \ + cupy_vertices.__cuda_array_interface__["data"][0] + cdef uintptr_t cupy_pageranks_ptr = \ + cupy_pageranks.__cuda_array_interface__["data"][0] + + return (cupy_vertices, cupy_pageranks) """ cdef cugraph_pagerank_result_t* my_result diff --git a/python/pylibcugraph/pylibcugraph/resource_handle.pxd b/python/pylibcugraph/pylibcugraph/resource_handle.pxd index afe6ba18a27..341110acc82 100644 --- a/python/pylibcugraph/pylibcugraph/resource_handle.pxd +++ b/python/pylibcugraph/pylibcugraph/resource_handle.pxd @@ -11,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 + from pylibcugraph._cugraph_c.cugraph_api cimport ( cugraph_resource_handle_t, ) diff --git a/python/pylibcugraph/pylibcugraph/resource_handle.pyx b/python/pylibcugraph/pylibcugraph/resource_handle.pyx index 30fc62ffd6d..7e78262bbc4 100644 --- a/python/pylibcugraph/pylibcugraph/resource_handle.pyx +++ b/python/pylibcugraph/pylibcugraph/resource_handle.pyx @@ -11,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 + from pylibcugraph._cugraph_c.cugraph_api cimport ( cugraph_create_resource_handle, cugraph_free_resource_handle, diff --git a/python/pylibcugraph/pylibcugraph/sssp.pyx b/python/pylibcugraph/pylibcugraph/sssp.pyx index 43e5d955e90..5867aa89e24 100644 --- a/python/pylibcugraph/pylibcugraph/sssp.pyx +++ b/python/pylibcugraph/pylibcugraph/sssp.pyx @@ -11,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 + from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, #data_type_id_t, @@ -63,21 +66,4 @@ def EXPERIMENTAL__sssp(EXPERIMENTAL__ResourceHandle resource_handle, &result_ptr, &error_ptr) - -""" - cdef cugraph_pagerank_result_t* my_result - cugraph_pagerank(... , &my_result, ...) - - cdef cugraph_type_erased_device_array_t* verts - cdef cugraph_type_erased_device_array_t* prs - verts = cugraph_pagerank_result_get_vertices(my_result) - prs = cugraph_pagerank_result_get_pageranks(my_result) - - do device-device copy on verts to user array - do device-device copy on prs to user array -/* -size_t cugraph_type_erased_device_array_size(const cugraph_type_erased_device_array_t* p); -data_type_id_t cugraph_type_erased_device_array_type(const cugraph_type_erased_device_array_t* p); -const void* cugraph_type_erased_device_array_pointer(const cugraph_type_erased_device_array_t* p); -*/ -""" + assert_success(error_code, error_ptr, "cugraph_sssp") diff --git a/python/pylibcugraph/pylibcugraph/utils.pxd b/python/pylibcugraph/pylibcugraph/utils.pxd index c2570156aeb..655f68e1163 100644 --- a/python/pylibcugraph/pylibcugraph/utils.pxd +++ b/python/pylibcugraph/pylibcugraph/utils.pxd @@ -11,6 +11,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 + +from pylibcugraph._cugraph_c.cugraph_api cimport ( + data_type_id_t, +) from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, cugraph_error_t, @@ -22,3 +28,5 @@ cdef assert_success(cugraph_error_code_t code, api_name) cdef assert_CAI_type(obj, var_name, allow_None=*) + +cdef get_numpy_type_from_c_type(data_type_id_t c_type) diff --git a/python/pylibcugraph/pylibcugraph/utils.pyx b/python/pylibcugraph/pylibcugraph/utils.pyx index 5d4c38294e1..ae65e0fc00c 100644 --- a/python/pylibcugraph/pylibcugraph/utils.pyx +++ b/python/pylibcugraph/pylibcugraph/utils.pyx @@ -11,6 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Have cython use python 3 syntax +# cython: language_level = 3 + +import numpy # FIXME: add tests for this cdef assert_success(cugraph_error_code_t code, @@ -31,3 +35,17 @@ cdef assert_CAI_type(obj, var_name, allow_None=False): if not(hasattr(obj, "__cuda_array_interface__")): raise TypeError(msg) + + +cdef get_numpy_type_from_c_type(data_type_id_t c_type): + if c_type == data_type_id_t.INT32: + return numpy.int32 + elif c_type == data_type_id_t.INT64: + return numpy.int64 + elif c_type == data_type_id_t.FLOAT32: + return numpy.float32 + elif c_type == data_type_id_t.FLOAT64: + return numpy.float64 + else: + raise RuntimeError("Internal error: got invalid data type enum value " + f"from C: {c_type}") From c32eee7a1c3609aca5d1192a8fbece35b2e7109c Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Mon, 24 Jan 2022 00:26:55 -0600 Subject: [PATCH 25/43] Updates for C API changes to support view types. --- .../pylibcugraph/_cugraph_c/algorithms.pxd | 28 +++---- .../pylibcugraph/_cugraph_c/array.pxd | 84 +++++++++++++++++-- .../pylibcugraph/_cugraph_c/graph.pxd | 20 ++--- python/pylibcugraph/pylibcugraph/graphs.pyx | 81 ++++++++---------- python/pylibcugraph/pylibcugraph/pagerank.pyx | 59 ++++++++++--- python/pylibcugraph/pylibcugraph/utils.pyx | 16 +++- 6 files changed, 194 insertions(+), 94 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd index 7d66caac39c..64a3d39933f 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd @@ -23,7 +23,7 @@ from pylibcugraph._cugraph_c.error cimport ( cugraph_error_t, ) from pylibcugraph._cugraph_c.array cimport ( - cugraph_type_erased_device_array_t, + cugraph_type_erased_device_array_view_t, ) from pylibcugraph._cugraph_c.graph cimport ( cugraph_graph_t, @@ -36,12 +36,12 @@ cdef extern from "cugraph_c/algorithms.h": ctypedef struct cugraph_pagerank_result_t: pass - cdef cugraph_type_erased_device_array_t* \ + cdef cugraph_type_erased_device_array_view_t* \ cugraph_pagerank_result_get_vertices( cugraph_pagerank_result_t* result ) - cdef cugraph_type_erased_device_array_t* \ + cdef cugraph_type_erased_device_array_view_t* \ cugraph_pagerank_result_get_pageranks( cugraph_pagerank_result_t* result ) @@ -55,7 +55,7 @@ cdef extern from "cugraph_c/algorithms.h": cugraph_pagerank( const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, - const cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, + const cugraph_type_erased_device_array_view_t* precomputed_vertex_out_weight_sums, double alpha, double epsilon, size_t max_iterations, @@ -69,9 +69,9 @@ cdef extern from "cugraph_c/algorithms.h": cugraph_personalized_pagerank( const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, - const cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums, - cugraph_type_erased_device_array_t* personalization_vertices, - const cugraph_type_erased_device_array_t* personalization_values, + const cugraph_type_erased_device_array_view_t* precomputed_vertex_out_weight_sums, + cugraph_type_erased_device_array_view_t* personalization_vertices, + const cugraph_type_erased_device_array_view_t* personalization_values, double alpha, double epsilon, size_t max_iterations, @@ -86,17 +86,17 @@ cdef extern from "cugraph_c/algorithms.h": ctypedef struct cugraph_paths_result_t: pass - cdef cugraph_type_erased_device_array_t* \ + cdef cugraph_type_erased_device_array_view_t* \ cugraph_paths_result_get_vertices( cugraph_paths_result_t* result ) - cdef cugraph_type_erased_device_array_t* \ + cdef cugraph_type_erased_device_array_view_t* \ cugraph_paths_result_get_distances( cugraph_paths_result_t* result ) - cdef cugraph_type_erased_device_array_t* \ + cdef cugraph_type_erased_device_array_view_t* \ cugraph_paths_result_get_predecessors( cugraph_paths_result_t* result ) @@ -113,9 +113,9 @@ cdef extern from "cugraph_c/algorithms.h": cugraph_extract_paths( const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, - const cugraph_type_erased_device_array_t* sources, + const cugraph_type_erased_device_array_view_t* sources, const cugraph_paths_result_t* paths_result, - const cugraph_type_erased_device_array_t* destinations, + const cugraph_type_erased_device_array_view_t* destinations, cugraph_extract_paths_result_t** result, cugraph_error_t** error ) @@ -125,7 +125,7 @@ cdef extern from "cugraph_c/algorithms.h": cugraph_extract_paths_result_t* result ) - cdef cugraph_type_erased_device_array_t* \ + cdef cugraph_type_erased_device_array_view_t* \ cugraph_extract_paths_result_get_paths( cugraph_extract_paths_result_t* result ) @@ -142,7 +142,7 @@ cdef extern from "cugraph_c/algorithms.h": const cugraph_resource_handle_t* handle, cugraph_graph_t* graph, # FIXME: this may become const - cugraph_type_erased_device_array_t* sources, + cugraph_type_erased_device_array_view_t* sources, bool_t direction_optimizing, size_t depth_limit, bool_t compute_predecessors, diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd index 711576c1366..91998d477d3 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/array.pxd @@ -30,9 +30,15 @@ cdef extern from "cugraph_c/array.h": ctypedef struct cugraph_type_erased_device_array_t: pass + ctypedef struct cugraph_type_erased_device_array_view_t: + pass + ctypedef struct cugraph_type_erased_host_array_t: pass + ctypedef struct cugraph_type_erased_host_array_view_t: + pass + cdef cugraph_error_code_t \ cugraph_type_erased_device_array_create( const cugraph_resource_handle_t* handle, @@ -47,14 +53,41 @@ cdef extern from "cugraph_c/array.h": cugraph_type_erased_device_array_t* p ) + # cdef void* \ + # cugraph_type_erased_device_array_release( + # cugraph_type_erased_device_array_t* p + # ) + + cdef cugraph_type_erased_device_array_view_t* \ + cugraph_type_erased_device_array_view( + cugraph_type_erased_device_array_t* array + ) + + cdef cugraph_type_erased_device_array_view_t* \ + cugraph_type_erased_device_array_view_create( + void* pointer, + size_t n_elems, + data_type_id_t dtype + ) + + cdef void \ + cugraph_type_erased_device_array_view_free( + cugraph_type_erased_device_array_view_t* p + ) + cdef size_t \ - cugraph_type_erased_device_array_size( - const cugraph_type_erased_device_array_t* p + cugraph_type_erased_device_array_view_size( + const cugraph_type_erased_device_array_view_t* p ) cdef data_type_id_t \ - cugraph_type_erased_device_array_type( - const cugraph_type_erased_device_array_t* p + cugraph_type_erased_device_array_view_type( + const cugraph_type_erased_device_array_view_t* p + ) + + cdef const void* \ + cugraph_type_erased_device_array_view_pointer( + const cugraph_type_erased_device_array_view_t* p ) cdef cugraph_error_code_t \ @@ -71,6 +104,28 @@ cdef extern from "cugraph_c/array.h": cugraph_type_erased_host_array_t* p ) + # cdef void* \ + # cugraph_type_erased_host_array_release( + # cugraph_type_erased_host_array_t* p + # ) + + cdef cugraph_type_erased_host_array_view_t* \ + cugraph_type_erased_host_array_view( + cugraph_type_erased_host_array_t* array + ) + + cdef cugraph_type_erased_host_array_view_t* \ + cugraph_type_erased_host_array_view_create( + void* pointer, + size_t n_elems, + data_type_id_t dtype + ) + + cdef void \ + cugraph_type_erased_host_array_view_free( + cugraph_type_erased_host_array_view_t* p + ) + cdef size_t \ cugraph_type_erased_host_array_size( const cugraph_type_erased_host_array_t* p @@ -81,18 +136,31 @@ cdef extern from "cugraph_c/array.h": const cugraph_type_erased_host_array_t* p ) + cdef void* \ + cugraph_type_erased_host_array_pointer( + const cugraph_type_erased_host_array_view_t* p + ) + cdef cugraph_error_code_t \ - cugraph_type_erased_device_array_copy_from_host( + cugraph_type_erased_device_array_view_copy_from_host( const cugraph_resource_handle_t* handle, - cugraph_type_erased_device_array_t* dst, + cugraph_type_erased_device_array_view_t* dst, const byte_t* h_src, cugraph_error_t** error ) cdef cugraph_error_code_t \ - cugraph_type_erased_device_array_copy_to_host( + cugraph_type_erased_device_array_view_copy_to_host( const cugraph_resource_handle_t* handle, byte_t* h_dst, - const cugraph_type_erased_device_array_t* src, + const cugraph_type_erased_device_array_view_t* src, + cugraph_error_t** error + ) + + cdef cugraph_error_code_t \ + cugraph_type_erased_device_array_view_copy( + const cugraph_resource_handle_t* handle, + cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* src, cugraph_error_t** error ) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd index c46e186b3cd..511b4a19048 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd @@ -23,8 +23,8 @@ from pylibcugraph._cugraph_c.error cimport ( cugraph_error_t, ) from pylibcugraph._cugraph_c.array cimport ( - cugraph_type_erased_device_array_t, - cugraph_type_erased_host_array_t, + cugraph_type_erased_device_array_view_t, + cugraph_type_erased_host_array_view_t, ) @@ -41,9 +41,9 @@ cdef extern from "cugraph_c/graph.h": cugraph_sg_graph_create( const cugraph_resource_handle_t* handle, const cugraph_graph_properties_t* properties, - const cugraph_type_erased_device_array_t* src, - const cugraph_type_erased_device_array_t* dst, - const cugraph_type_erased_device_array_t* weights, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, bool_t store_transposed, bool_t renumber, bool_t check, @@ -60,11 +60,11 @@ cdef extern from "cugraph_c/graph.h": cugraph_mg_graph_create( const cugraph_resource_handle_t* handle, const cugraph_graph_properties_t* properties, - const cugraph_type_erased_device_array_t* src, - const cugraph_type_erased_device_array_t* dst, - const cugraph_type_erased_device_array_t* weights, - const cugraph_type_erased_host_array_t* vertex_partition_offsets, - const cugraph_type_erased_host_array_t* segment_offsets, + const cugraph_type_erased_device_array_view_t* src, + const cugraph_type_erased_device_array_view_t* dst, + const cugraph_type_erased_device_array_view_t* weights, + const cugraph_type_erased_host_array_view_t* vertex_partition_offsets, + const cugraph_type_erased_host_array_view_t* segment_offsets, bool_t store_transposed, size_t num_vertices, size_t num_edges, diff --git a/python/pylibcugraph/pylibcugraph/graphs.pyx b/python/pylibcugraph/pylibcugraph/graphs.pyx index 5274cdcdeec..0f0494d3919 100644 --- a/python/pylibcugraph/pylibcugraph/graphs.pyx +++ b/python/pylibcugraph/pylibcugraph/graphs.pyx @@ -14,6 +14,8 @@ # Have cython use python 3 syntax # cython: language_level = 3 +from libc.stdint cimport uintptr_t + from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, cugraph_resource_handle_t, @@ -24,8 +26,8 @@ from pylibcugraph._cugraph_c.error cimport ( cugraph_error_t, ) from pylibcugraph._cugraph_c.array cimport ( - cugraph_type_erased_device_array_t, - cugraph_type_erased_device_array_create, + cugraph_type_erased_device_array_view_t, + cugraph_type_erased_device_array_view_create, cugraph_type_erased_device_array_free, ) from pylibcugraph._cugraph_c.graph cimport ( @@ -78,67 +80,50 @@ cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): cdef cugraph_error_t* error_ptr cdef cugraph_error_code_t error_code - cdef cugraph_type_erased_device_array_t* srcs_ptr - cdef cugraph_type_erased_device_array_t* dsts_ptr - cdef cugraph_type_erased_device_array_t* weights_ptr - # FIXME: set dtype properly - error_code = cugraph_type_erased_device_array_create( - resource_handle.c_resource_handle_ptr, - data_type_id_t.INT32, - len(src_array), - &srcs_ptr, - &error_ptr) - - assert_success(error_code, error_ptr, - "cugraph_type_erased_device_array_create()") - # FIXME: add call to to device-device copy of __cuda_array_interface__ - # values to cugraph_type_erased_device_array - # FIXME: don't do above fixme, use device view API + cdef uintptr_t cai_srcs_ptr = \ + src_array.__cuda_array_interface__["data"][0] + cdef cugraph_type_erased_device_array_view_t* srcs_view_ptr = \ + cugraph_type_erased_device_array_view_create( + cai_srcs_ptr, + len(src_array), + data_type_id_t.INT32) # FIXME: set dtype properly - error_code = cugraph_type_erased_device_array_create( - resource_handle.c_resource_handle_ptr, - data_type_id_t.INT32, - len(dst_array), - &dsts_ptr, - &error_ptr) - - assert_success(error_code, error_ptr, - "cugraph_type_erased_device_array_create()") - # FIXME: add call to to device-device copy of __cuda_array_interface__ - # values to cugraph_type_erased_device_array - # FIXME: don't do above fixme, use device view API + cdef uintptr_t cai_dsts_ptr = \ + dst_array.__cuda_array_interface__["data"][0] + cdef cugraph_type_erased_device_array_view_t* dsts_view_ptr = \ + cugraph_type_erased_device_array_view_create( + cai_dsts_ptr, + len(dst_array), + data_type_id_t.INT32) # FIXME: set dtype properly - error_code = cugraph_type_erased_device_array_create( - resource_handle.c_resource_handle_ptr, - data_type_id_t.FLOAT32, - len(weight_array), - &weights_ptr, - &error_ptr) - - assert_success(error_code, error_ptr, - "cugraph_type_erased_device_array_create()") - # FIXME: add call to to device-device copy of __cuda_array_interface__ - # values to cugraph_type_erased_device_array - # FIXME: don't do above fixme, use device view API + cdef uintptr_t cai_weights_ptr = \ + weight_array.__cuda_array_interface__["data"][0] + cdef cugraph_type_erased_device_array_view_t* weights_view_ptr = \ + cugraph_type_erased_device_array_view_create( + cai_weights_ptr, + len(weight_array), + data_type_id_t.FLOAT32) error_code = cugraph_sg_graph_create( resource_handle.c_resource_handle_ptr, &(graph_properties.c_graph_properties), - srcs_ptr, - dsts_ptr, - weights_ptr, - int(store_transposed), - int(renumber), - int(expensive_check), + srcs_view_ptr, + dsts_view_ptr, + weights_view_ptr, + store_transposed, + renumber, + expensive_check, &(self.c_graph_ptr), &error_ptr) assert_success(error_code, error_ptr, "cugraph_sg_graph_create()") + # FIXME: free the views + def __dealloc__(self): if self.c_graph_ptr is not NULL: cugraph_sg_graph_free(self.c_graph_ptr) diff --git a/python/pylibcugraph/pylibcugraph/pagerank.pyx b/python/pylibcugraph/pylibcugraph/pagerank.pyx index 94035b2205f..08a2fe5f620 100644 --- a/python/pylibcugraph/pylibcugraph/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/pagerank.pyx @@ -26,11 +26,12 @@ from pylibcugraph._cugraph_c.error cimport ( cugraph_error_t, ) from pylibcugraph._cugraph_c.array cimport ( - cugraph_type_erased_device_array_t, - cugraph_type_erased_device_array_size, - cugraph_type_erased_device_array_type, - cugraph_type_erased_device_array_create, - cugraph_type_erased_device_array_free, + cugraph_type_erased_device_array_view_t, + cugraph_type_erased_device_array_view_size, + cugraph_type_erased_device_array_view_type, + cugraph_type_erased_device_array_view_create, + cugraph_type_erased_device_array_view_free, + cugraph_type_erased_device_array_view_copy, ) from pylibcugraph._cugraph_c.graph cimport ( cugraph_graph_t, @@ -40,6 +41,7 @@ from pylibcugraph._cugraph_c.algorithms cimport ( cugraph_pagerank, cugraph_pagerank_result_get_vertices, cugraph_pagerank_result_get_pageranks, + cugraph_pagerank_result_free, ) from pylibcugraph.resource_handle cimport ( @@ -74,7 +76,7 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, cdef cugraph_resource_handle_t* c_resource_handle_ptr = resource_handle.c_resource_handle_ptr cdef cugraph_graph_t* c_graph_ptr = graph.c_graph_ptr - cdef cugraph_type_erased_device_array_t* precomputed_vertex_out_weight_sums_ptr = NULL + cdef cugraph_type_erased_device_array_view_t* precomputed_vertex_out_weight_sums_ptr = NULL if precomputed_vertex_out_weight_sums: raise NotImplementedError("None is temporarily the only supported value for precomputed_vertex_out_weight_sums") #precomputed_vertex_out_weight_sums_ptr = precomputed_vertex_out_weight_sums @@ -97,17 +99,20 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, assert_success(error_code, error_ptr, "cugraph_pagerank") # Extract individual device array pointers from result - cdef cugraph_type_erased_device_array_t* vertices_ptr - cdef cugraph_type_erased_device_array_t* pageranks_ptr + cdef cugraph_type_erased_device_array_view_t* vertices_ptr + cdef cugraph_type_erased_device_array_view_t* pageranks_ptr vertices_ptr = cugraph_pagerank_result_get_vertices(result_ptr) pageranks_ptr = cugraph_pagerank_result_get_pageranks(result_ptr) # Extract meta-data needed to copy results - vertex_numpy_type = get_numpy_type_from_c_type( - cugraph_type_erased_device_array_type(vertices_ptr)) - pagerank_numpy_type = get_numpy_type_from_c_type( - cugraph_type_erased_device_array_type(pageranks_ptr)) - num_vertices = cugraph_type_erased_device_array_size(vertices_ptr) + cdef data_type_id_t vertex_type = \ + cugraph_type_erased_device_array_view_type(vertices_ptr) + cdef data_type_id_t pagerank_type = \ + cugraph_type_erased_device_array_view_type(pageranks_ptr) + vertex_numpy_type = get_numpy_type_from_c_type(vertex_type) + pagerank_numpy_type = get_numpy_type_from_c_type(pagerank_type) + + num_vertices = cugraph_type_erased_device_array_view_size(vertices_ptr) # Set up cupy arrays to return and copy results to those cupy_vertices = cupy.array(numpy.zeros(num_vertices), @@ -120,9 +125,37 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, cdef uintptr_t cupy_pageranks_ptr = \ cupy_pageranks.__cuda_array_interface__["data"][0] + cdef cugraph_type_erased_device_array_view_t* cupy_vertices_view_ptr = \ + cugraph_type_erased_device_array_view_create( + cupy_vertices_ptr, num_vertices, vertex_type) + + cdef cugraph_type_erased_device_array_view_t* cupy_pageranks_view_ptr = \ + cugraph_type_erased_device_array_view_create( + cupy_pageranks_ptr, num_vertices, vertex_type) + + # FIXME: free the views somewhere + + error_code = cugraph_type_erased_device_array_view_copy( + c_resource_handle_ptr, + cupy_vertices_view_ptr, + vertices_ptr, + &error_ptr) + assert_success(error_code, error_ptr, + "cugraph_type_erased_device_array_view_copy") + error_code = cugraph_type_erased_device_array_view_copy( + c_resource_handle_ptr, + cupy_pageranks_view_ptr, + pageranks_ptr, + &error_ptr) + assert_success(error_code, error_ptr, + "cugraph_type_erased_device_array_view_copy") + + cugraph_pagerank_result_free(result_ptr) return (cupy_vertices, cupy_pageranks) + + """ cdef cugraph_pagerank_result_t* my_result cugraph_pagerank(... , &my_result, ...) diff --git a/python/pylibcugraph/pylibcugraph/utils.pyx b/python/pylibcugraph/pylibcugraph/utils.pyx index ae65e0fc00c..dc154e3fa85 100644 --- a/python/pylibcugraph/pylibcugraph/utils.pyx +++ b/python/pylibcugraph/pylibcugraph/utils.pyx @@ -21,8 +21,22 @@ cdef assert_success(cugraph_error_code_t code, cugraph_error_t* err, api_name): if code != cugraph_error_code_t.CUGRAPH_SUCCESS: + if code == cugraph_error_code_t.CUGRAPH_UNKNOWN_ERROR: + code_str = "CUGRAPH_UNKNOWN_ERROR" + elif code == cugraph_error_code_t.CUGRAPH_INVALID_HANDLE: + code_str = "CUGRAPH_INVALID_HANDLE" + elif code == cugraph_error_code_t.CUGRAPH_ALLOC_ERROR: + code_str = "CUGRAPH_ALLOC_ERROR" + elif code == cugraph_error_code_t.CUGRAPH_INVALID_INPUT: + code_str = "CUGRAPH_INVALID_INPUT" + elif code == cugraph_error_code_t.CUGRAPH_NOT_IMPLEMENTED: + code_str = "CUGRAPH_NOT_IMPLEMENTED" + elif code == cugraph_error_code_t.CUGRAPH_UNSUPPORTED_TYPE_COMBINATION: + code_str = "CUGRAPH_UNSUPPORTED_TYPE_COMBINATION" + else: + code_str = "unknown error code" # FIXME: extract message using cugraph_error_message() - raise RuntimeError(f"non-success value returned from {api_name}") + raise RuntimeError(f"non-success value returned from {api_name}: {code_str}") cdef assert_CAI_type(obj, var_name, allow_None=False): From 931e6e138e88163b20d66f8f5cb4cb9ea5879ecc Mon Sep 17 00:00:00 2001 From: Chuck Hastings Date: Mon, 24 Jan 2022 11:46:12 -0500 Subject: [PATCH 26/43] generate the view correctly for extract paths --- cpp/src/c_api/extract_paths.cpp | 2 +- cpp/tests/c_api/extract_paths_test.c | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cpp/src/c_api/extract_paths.cpp b/cpp/src/c_api/extract_paths.cpp index 4772bc02958..f22896e8e1b 100644 --- a/cpp/src/c_api/extract_paths.cpp +++ b/cpp/src/c_api/extract_paths.cpp @@ -151,7 +151,7 @@ cugraph_type_erased_device_array_view_t* cugraph_extract_paths_result_get_paths( cugraph_extract_paths_result_t* result) { auto internal_pointer = reinterpret_cast(result); - return reinterpret_cast(internal_pointer->paths_); + return reinterpret_cast(internal_pointer->paths_->view()); } extern "C" void cugraph_extract_paths_result_free(cugraph_extract_paths_result_t* result) diff --git a/cpp/tests/c_api/extract_paths_test.c b/cpp/tests/c_api/extract_paths_test.c index 6af358547da..919b3d830ed 100644 --- a/cpp/tests/c_api/extract_paths_test.c +++ b/cpp/tests/c_api/extract_paths_test.c @@ -99,11 +99,11 @@ int generic_bfs_test_with_extract_paths(vertex_t* h_src, &ret_error); size_t max_path_length = cugraph_extract_paths_result_get_max_path_length(p_extract_paths_result); - cugraph_type_erased_device_array_t* paths = - cugraph_extract_paths_result_get_paths(p_extract_paths_result); + TEST_ASSERT( + test_ret_value, max_path_length == expected_max_path_length, "path lengths don't match"); cugraph_type_erased_device_array_view_t* paths_view = - cugraph_type_erased_device_array_view(paths); + cugraph_extract_paths_result_get_paths(p_extract_paths_result); size_t paths_size = cugraph_type_erased_device_array_view_size(paths_view); @@ -117,12 +117,11 @@ int generic_bfs_test_with_extract_paths(vertex_t* h_src, TEST_ASSERT(test_ret_value, expected_paths[i] == h_paths[i], "paths don't match"); } + cugraph_type_erased_device_array_view_free(paths_view); cugraph_type_erased_device_array_view_free(p_sources_view); cugraph_type_erased_device_array_view_free(p_destinations_view); - cugraph_type_erased_device_array_view_free(paths_view); cugraph_type_erased_device_array_free(p_sources); cugraph_type_erased_device_array_free(p_destinations); - cugraph_type_erased_device_array_free(paths); cugraph_extract_paths_result_free(p_extract_paths_result); cugraph_paths_result_free(p_paths_result); cugraph_sg_graph_free(p_graph); From 2c0cda6b09179cb6ce5d676a117e8a0bbbe0657e Mon Sep 17 00:00:00 2001 From: Chuck Hastings Date: Mon, 24 Jan 2022 11:49:19 -0500 Subject: [PATCH 27/43] should use raft::copy in the device copy --- cpp/src/c_api/array.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/src/c_api/array.cpp b/cpp/src/c_api/array.cpp index b5ac0285f8f..bbf5741d8d3 100644 --- a/cpp/src/c_api/array.cpp +++ b/cpp/src/c_api/array.cpp @@ -314,10 +314,10 @@ extern "C" cugraph_error_code_t cugraph_type_erased_device_array_view_copy( return CUGRAPH_INVALID_INPUT; } - raft::update_host(reinterpret_cast(internal_pointer_dst->data_), - reinterpret_cast(internal_pointer_src->data_), - internal_pointer_src->num_bytes(), - raft_handle->get_stream()); + raft::copy(reinterpret_cast(internal_pointer_dst->data_), + reinterpret_cast(internal_pointer_src->data_), + internal_pointer_src->num_bytes(), + raft_handle->get_stream()); return CUGRAPH_SUCCESS; } catch (std::exception const& ex) { From 720ffa5279643f1400a13527d14af6828476cf21 Mon Sep 17 00:00:00 2001 From: Chuck Hastings Date: Mon, 24 Jan 2022 12:54:00 -0500 Subject: [PATCH 28/43] fix clang-format issues --- cpp/src/c_api/extract_paths.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/src/c_api/extract_paths.cpp b/cpp/src/c_api/extract_paths.cpp index f22896e8e1b..35b3eba355d 100644 --- a/cpp/src/c_api/extract_paths.cpp +++ b/cpp/src/c_api/extract_paths.cpp @@ -151,7 +151,8 @@ cugraph_type_erased_device_array_view_t* cugraph_extract_paths_result_get_paths( cugraph_extract_paths_result_t* result) { auto internal_pointer = reinterpret_cast(result); - return reinterpret_cast(internal_pointer->paths_->view()); + return reinterpret_cast( + internal_pointer->paths_->view()); } extern "C" void cugraph_extract_paths_result_free(cugraph_extract_paths_result_t* result) From e47c8033bcb1bab1d909d6206c6f60d26c29dd25 Mon Sep 17 00:00:00 2001 From: Chuck Hastings Date: Mon, 24 Jan 2022 20:44:44 -0500 Subject: [PATCH 29/43] fix a few bugs that Rick found --- cpp/src/c_api/array.cpp | 5 +++-- cpp/src/c_api/graph_sg.cpp | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cpp/src/c_api/array.cpp b/cpp/src/c_api/array.cpp index bbf5741d8d3..c6e6d8ac262 100644 --- a/cpp/src/c_api/array.cpp +++ b/cpp/src/c_api/array.cpp @@ -93,7 +93,8 @@ cugraph_type_erased_device_array_view_t* cugraph_type_erased_device_array_view_c void* pointer, size_t n_elems, data_type_id_t dtype) { return reinterpret_cast( - new cugraph::c_api::cugraph_type_erased_device_array_view_t{pointer, n_elems, dtype}); + new cugraph::c_api::cugraph_type_erased_device_array_view_t{ + pointer, n_elems, n_elems * (::data_type_sz[dtype]), dtype}); } extern "C" void cugraph_type_erased_device_array_view_free( @@ -189,7 +190,7 @@ extern "C" cugraph_type_erased_host_array_view_t* cugraph_type_erased_host_array { return reinterpret_cast( new cugraph::c_api::cugraph_type_erased_host_array_view_t{ - static_cast(pointer), n_elems, dtype}); + static_cast(pointer), n_elems, n_elems * (::data_type_sz[dtype]), dtype}); } extern "C" void cugraph_type_erased_host_array_view_free(cugraph_type_erased_host_array_view_t* p) diff --git a/cpp/src/c_api/graph_sg.cpp b/cpp/src/c_api/graph_sg.cpp index 61b14129dd0..bf94003e9f5 100644 --- a/cpp/src/c_api/graph_sg.cpp +++ b/cpp/src/c_api/graph_sg.cpp @@ -26,6 +26,8 @@ #include +#include + namespace cugraph { namespace c_api { @@ -183,7 +185,7 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( cugraph_error_t** error) { constexpr bool multi_gpu = false; - constexpr size_t int32_threshold{2 ^ 31 - 1}; + constexpr size_t int32_threshold{std::numeric_limits::max()}; *graph = nullptr; *error = nullptr; From dd27db16ee4bd2b4bc1d2cb1ac4125c095797111 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Mon, 24 Jan 2022 21:25:05 -0600 Subject: [PATCH 30/43] Initial passing pagerank tests. --- .../pylibcugraph/_cugraph_c/cugraph_api.pxd | 1 - .../pylibcugraph/tests/conftest.py | 69 +++++++++++++++---- .../pylibcugraph/tests/test_pagerank.py | 25 +++++-- 3 files changed, 77 insertions(+), 18 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd index 0d3b0fc0185..276b8062849 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/cugraph_api.pxd @@ -28,7 +28,6 @@ cdef extern from "cugraph_c/cugraph_api.h": INT64 FLOAT32 FLOAT64 - NTYPES ctypedef int8_t byte_t diff --git a/python/pylibcugraph/pylibcugraph/tests/conftest.py b/python/pylibcugraph/pylibcugraph/tests/conftest.py index 35f59bec748..f9a66abb321 100644 --- a/python/pylibcugraph/pylibcugraph/tests/conftest.py +++ b/python/pylibcugraph/pylibcugraph/tests/conftest.py @@ -107,6 +107,29 @@ def get_graph_data_for_dataset(ds, ds_name): return (device_srcs, device_dsts, device_weights, ds_name, is_valid) +def create_SGGraph(resource_handle, + device_srcs, + device_dsts, + device_weights, + transposed=True): + """ + Creates and returns a SGGraph instance using the parameters passed in. + """ + from pylibcugraph.experimental import (SGGraph, + GraphProperties, + ) + graph_props = GraphProperties(is_symmetric=False, is_multigraph=False) + + return SGGraph(resource_handle, + graph_props, + device_srcs, + device_dsts, + device_weights, + store_transposed=transposed, + renumber=False, + expensive_check=False) + + # ============================================================================= # Pytest fixtures # ============================================================================= @@ -140,10 +163,34 @@ def sg_graph_objs(valid_graph_data, request): the associated resource handle, and the name of the dataset used to construct the graph. """ - from pylibcugraph.experimental import (SGGraph, - ResourceHandle, - GraphProperties, - ) + from pylibcugraph.experimental import ResourceHandle + + (device_srcs, device_dsts, device_weights, ds_name, is_valid) = \ + valid_graph_data + + if is_valid is False: + pytest.exit("got invalid graph data - expecting only valid data") + + resource_handle = ResourceHandle() + g = create_SGGraph(resource_handle, + device_srcs, + device_dsts, + device_weights, + transposed=False) + + return (g, resource_handle, ds_name) + + +@pytest.fixture(scope="package") +def sg_transposed_graph_objs(valid_graph_data, request): + """ + Returns a tuple containing the SGGraph object constructed from + parameterized values returned by the valid_graph_data fixture, + the associated resource handle, and the name of the dataset + used to construct the graph. + The SGGraph object is created with the transposed arg set to True. + """ + from pylibcugraph.experimental import ResourceHandle (device_srcs, device_dsts, device_weights, ds_name, is_valid) = \ valid_graph_data @@ -151,15 +198,11 @@ def sg_graph_objs(valid_graph_data, request): if is_valid is False: pytest.exit("got invalid graph data - expecting only valid data") - graph_props = GraphProperties(is_symmetric=False, is_multigraph=False) resource_handle = ResourceHandle() + g = create_SGGraph(resource_handle, + device_srcs, + device_dsts, + device_weights, + transposed=True) - g = SGGraph(resource_handle, - graph_props, - device_srcs, - device_dsts, - device_weights, - store_transposed=False, - renumber=False, - expensive_check=False) return (g, resource_handle, ds_name) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py index 34bd62f2859..ef2839317c0 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py @@ -89,11 +89,10 @@ # ============================================================================= # Tests # ============================================================================= -@pytest.mark.skip(reason="UNFINISHED") -def test_pagerank(sg_graph_objs): +def test_pagerank(sg_transposed_graph_objs): from pylibcugraph.experimental import pagerank - (g, resource_handle, ds_name) = sg_graph_objs + (g, resource_handle, ds_name) = sg_transposed_graph_objs expected_result = _test_data[ds_name] precomputed_vertex_out_weight_sums = None @@ -109,4 +108,22 @@ def test_pagerank(sg_graph_objs): has_initial_guess, do_expensive_check) - assert result == expected_result + (expected_verts, expected_pageranks) = expected_result + num_expected_verts = len(expected_verts) + (actual_verts, actual_pageranks) = result + + # Do a simple check using the vertices as array indices. First, ensure + # the test data vertices start from 0 with no gaps. + assert sum(range(num_expected_verts)) == sum(expected_verts) + + assert actual_verts.dtype == expected_verts.dtype + assert actual_pageranks.dtype == expected_pageranks.dtype + + actual_pageranks = actual_pageranks.tolist() + actual_verts = actual_verts.tolist() + expected_pageranks = expected_pageranks.tolist() + + for i in range(num_expected_verts): + assert actual_pageranks[i] == \ + pytest.approx(expected_pageranks[actual_verts[i]], 1e-4), \ + f"actual != expected for result at index {i}" From 90c3a04409d8474660c853ea9fd91caa455c0bad Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 25 Jan 2022 14:13:43 -0600 Subject: [PATCH 31/43] Test refactoring, finished initial SSSP and updated tests. --- python/pylibcugraph/pylibcugraph/pagerank.pyx | 2 +- python/pylibcugraph/pylibcugraph/sssp.pyx | 97 ++++++++++++++++++- .../pylibcugraph/tests/conftest.py | 41 ++++---- .../pylibcugraph/tests/test_pagerank.py | 12 +-- .../pylibcugraph/tests/test_sssp.py | 46 +++++++-- 5 files changed, 159 insertions(+), 39 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/pagerank.pyx b/python/pylibcugraph/pylibcugraph/pagerank.pyx index 08a2fe5f620..ff8ad22a9b6 100644 --- a/python/pylibcugraph/pylibcugraph/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/pagerank.pyx @@ -114,7 +114,7 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, num_vertices = cugraph_type_erased_device_array_view_size(vertices_ptr) - # Set up cupy arrays to return and copy results to those + # Set up cupy arrays to return and copy results cupy_vertices = cupy.array(numpy.zeros(num_vertices), dtype=vertex_numpy_type) cupy_pageranks = cupy.array(numpy.zeros(num_vertices), diff --git a/python/pylibcugraph/pylibcugraph/sssp.pyx b/python/pylibcugraph/pylibcugraph/sssp.pyx index 5867aa89e24..f5f3b909935 100644 --- a/python/pylibcugraph/pylibcugraph/sssp.pyx +++ b/python/pylibcugraph/pylibcugraph/sssp.pyx @@ -14,21 +14,35 @@ # Have cython use python 3 syntax # cython: language_level = 3 +from libc.stdint cimport uintptr_t + from pylibcugraph._cugraph_c.cugraph_api cimport ( bool_t, - #data_type_id_t, + data_type_id_t, cugraph_resource_handle_t, ) from pylibcugraph._cugraph_c.error cimport ( cugraph_error_code_t, cugraph_error_t, ) +from pylibcugraph._cugraph_c.array cimport ( + cugraph_type_erased_device_array_view_t, + cugraph_type_erased_device_array_view_size, + cugraph_type_erased_device_array_view_type, + cugraph_type_erased_device_array_view_create, + cugraph_type_erased_device_array_view_free, + cugraph_type_erased_device_array_view_copy, +) from pylibcugraph._cugraph_c.graph cimport ( cugraph_graph_t, ) from pylibcugraph._cugraph_c.algorithms cimport ( - cugraph_paths_result_t, cugraph_sssp, + cugraph_paths_result_t, + cugraph_paths_result_get_vertices, + cugraph_paths_result_get_distances, + cugraph_paths_result_get_predecessors, + cugraph_paths_result_free, ) from pylibcugraph.resource_handle cimport ( @@ -39,8 +53,12 @@ from pylibcugraph.graphs cimport ( ) from pylibcugraph.utils cimport ( assert_success, + get_numpy_type_from_c_type, ) +import cupy +import numpy + def EXPERIMENTAL__sssp(EXPERIMENTAL__ResourceHandle resource_handle, EXPERIMENTAL__Graph graph, @@ -65,5 +83,78 @@ def EXPERIMENTAL__sssp(EXPERIMENTAL__ResourceHandle resource_handle, do_expensive_check, &result_ptr, &error_ptr) - assert_success(error_code, error_ptr, "cugraph_sssp") + + # Extract individual device array pointers from result + cdef cugraph_type_erased_device_array_view_t* vertices_ptr + cdef cugraph_type_erased_device_array_view_t* distances_ptr + cdef cugraph_type_erased_device_array_view_t* predecessors_ptr + vertices_ptr = cugraph_paths_result_get_vertices(result_ptr) + distances_ptr = cugraph_paths_result_get_distances(result_ptr) + predecessors_ptr = cugraph_paths_result_get_predecessors(result_ptr) + + # Extract meta-data needed to copy results + cdef data_type_id_t vertex_type = \ + cugraph_type_erased_device_array_view_type(vertices_ptr) + cdef data_type_id_t distance_type = \ + cugraph_type_erased_device_array_view_type(distances_ptr) + cdef data_type_id_t predecessor_type = \ + cugraph_type_erased_device_array_view_type(predecessors_ptr) + vertex_numpy_type = get_numpy_type_from_c_type(vertex_type) + distance_numpy_type = get_numpy_type_from_c_type(distance_type) + predecessor_numpy_type = get_numpy_type_from_c_type(predecessor_type) + + num_vertices = cugraph_type_erased_device_array_view_size(vertices_ptr) + + # Set up cupy arrays to return and copy results + cupy_vertices = cupy.array(numpy.zeros(num_vertices), + dtype=vertex_numpy_type) + cupy_distances = cupy.array(numpy.zeros(num_vertices), + dtype=distance_numpy_type) + cupy_predecessors = cupy.array(numpy.zeros(num_vertices), + dtype=predecessor_numpy_type) + + cdef uintptr_t cupy_vertices_ptr = \ + cupy_vertices.__cuda_array_interface__["data"][0] + cdef uintptr_t cupy_distances_ptr = \ + cupy_distances.__cuda_array_interface__["data"][0] + cdef uintptr_t cupy_predecessors_ptr = \ + cupy_predecessors.__cuda_array_interface__["data"][0] + + cdef cugraph_type_erased_device_array_view_t* cupy_vertices_view_ptr = \ + cugraph_type_erased_device_array_view_create( + cupy_vertices_ptr, num_vertices, vertex_type) + cdef cugraph_type_erased_device_array_view_t* cupy_distances_view_ptr = \ + cugraph_type_erased_device_array_view_create( + cupy_distances_ptr, num_vertices, vertex_type) + cdef cugraph_type_erased_device_array_view_t* cupy_predecessors_view_ptr = \ + cugraph_type_erased_device_array_view_create( + cupy_predecessors_ptr, num_vertices, vertex_type) + + # FIXME: free the views somewhere + + error_code = cugraph_type_erased_device_array_view_copy( + c_resource_handle_ptr, + cupy_vertices_view_ptr, + vertices_ptr, + &error_ptr) + assert_success(error_code, error_ptr, + "cugraph_type_erased_device_array_view_copy") + error_code = cugraph_type_erased_device_array_view_copy( + c_resource_handle_ptr, + cupy_distances_view_ptr, + distances_ptr, + &error_ptr) + assert_success(error_code, error_ptr, + "cugraph_type_erased_device_array_view_copy") + error_code = cugraph_type_erased_device_array_view_copy( + c_resource_handle_ptr, + cupy_predecessors_view_ptr, + predecessors_ptr, + &error_ptr) + assert_success(error_code, error_ptr, + "cugraph_type_erased_device_array_view_copy") + + cugraph_paths_result_free(result_ptr) + + return (cupy_vertices, cupy_distances, cupy_predecessors) diff --git a/python/pylibcugraph/pylibcugraph/tests/conftest.py b/python/pylibcugraph/pylibcugraph/tests/conftest.py index f9a66abb321..89d8fa57843 100644 --- a/python/pylibcugraph/pylibcugraph/tests/conftest.py +++ b/python/pylibcugraph/pylibcugraph/tests/conftest.py @@ -107,28 +107,31 @@ def get_graph_data_for_dataset(ds, ds_name): return (device_srcs, device_dsts, device_weights, ds_name, is_valid) -def create_SGGraph(resource_handle, - device_srcs, +def create_SGGraph(device_srcs, device_dsts, device_weights, - transposed=True): + transposed=False): """ - Creates and returns a SGGraph instance using the parameters passed in. + Creates and returns a SGGraph instance and the corresponding ResourceHandle + using the parameters passed in. """ from pylibcugraph.experimental import (SGGraph, + ResourceHandle, GraphProperties, ) + resource_handle = ResourceHandle() graph_props = GraphProperties(is_symmetric=False, is_multigraph=False) - return SGGraph(resource_handle, - graph_props, - device_srcs, - device_dsts, - device_weights, - store_transposed=transposed, - renumber=False, - expensive_check=False) + g = SGGraph(resource_handle, + graph_props, + device_srcs, + device_dsts, + device_weights, + store_transposed=transposed, + renumber=False, + expensive_check=False) + return (g, resource_handle) # ============================================================================= # Pytest fixtures @@ -163,17 +166,14 @@ def sg_graph_objs(valid_graph_data, request): the associated resource handle, and the name of the dataset used to construct the graph. """ - from pylibcugraph.experimental import ResourceHandle - (device_srcs, device_dsts, device_weights, ds_name, is_valid) = \ valid_graph_data if is_valid is False: pytest.exit("got invalid graph data - expecting only valid data") - resource_handle = ResourceHandle() - g = create_SGGraph(resource_handle, - device_srcs, + (g, resource_handle) = \ + create_SGGraph(device_srcs, device_dsts, device_weights, transposed=False) @@ -190,17 +190,14 @@ def sg_transposed_graph_objs(valid_graph_data, request): used to construct the graph. The SGGraph object is created with the transposed arg set to True. """ - from pylibcugraph.experimental import ResourceHandle - (device_srcs, device_dsts, device_weights, ds_name, is_valid) = \ valid_graph_data if is_valid is False: pytest.exit("got invalid graph data - expecting only valid data") - resource_handle = ResourceHandle() - g = create_SGGraph(resource_handle, - device_srcs, + (g, resource_handle) = \ + create_SGGraph(device_srcs, device_dsts, device_weights, transposed=True) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py index ef2839317c0..76c88d53eb6 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py @@ -82,18 +82,19 @@ # ============================================================================= -# Helper functions +# Tests # ============================================================================= +# FIXME: add tests for non-transposed graphs too, which should either work (via +# auto-transposing in C) or raise the appropriate exception. -# ============================================================================= -# Tests -# ============================================================================= def test_pagerank(sg_transposed_graph_objs): +#def test_pagerank(sg_graph_objs): from pylibcugraph.experimental import pagerank (g, resource_handle, ds_name) = sg_transposed_graph_objs - expected_result = _test_data[ds_name] + #(g, resource_handle, ds_name) = sg_graph_objs + (expected_verts, expected_pageranks) = _test_data[ds_name] precomputed_vertex_out_weight_sums = None has_initial_guess = False @@ -108,7 +109,6 @@ def test_pagerank(sg_transposed_graph_objs): has_initial_guess, do_expensive_check) - (expected_verts, expected_pageranks) = expected_result num_expected_verts = len(expected_verts) (actual_verts, actual_pageranks) = result diff --git a/python/pylibcugraph/pylibcugraph/tests/test_sssp.py b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py index fad071f6f41..8d7a9cd2a30 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_sssp.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py @@ -60,7 +60,7 @@ }, "Simple_1": { "start_vertex": 1, - "vertex": cp.asarray(range(34), dtype=np.int32), + "vertex": cp.asarray(range(4), dtype=np.int32), "distance": cp.asarray( [3.4028235e+38, 0.0000000e+00, 1.0000000e+00, 2.0000000e+00, @@ -73,7 +73,7 @@ }, "Simple_2": { "start_vertex": 1, - "vertex": cp.asarray(range(34), dtype=np.int32), + "vertex": cp.asarray(range(6), dtype=np.int32), "distance": cp.asarray( [3.4028235e+38, 0.0000000e+00, 3.4028235e+38, 2.0999999e+00, 1.1000000e+00, 4.3000002e+00 @@ -100,14 +100,13 @@ # ============================================================================= # Tests # ============================================================================= -@pytest.mark.skip(reason="UNFINISHED") def test_sssp(sg_graph_objs): from pylibcugraph.experimental import sssp (g, resource_handle, ds_name) = sg_graph_objs (source, - expected_vertices, + expected_verts, expected_distances, expected_predecessors) = _test_data[ds_name].values() @@ -122,6 +121,39 @@ def test_sssp(sg_graph_objs): compute_predecessors, do_expensive_check) - assert result == (expected_vertices, - expected_distances, - expected_predecessors) + num_expected_verts = len(expected_verts) + (actual_verts, actual_distances, actual_predecessors) = result + + # Do a simple check using the vertices as array indices. First, ensure + # the test data vertices start from 0 with no gaps. + assert sum(range(num_expected_verts)) == sum(expected_verts) + + assert actual_verts.dtype == expected_verts.dtype + assert actual_distances.dtype == expected_distances.dtype + assert actual_predecessors.dtype == expected_predecessors.dtype + + actual_verts = actual_verts.tolist() + actual_distances = actual_distances.tolist() + actual_predecessors = actual_predecessors.tolist() + expected_distances = expected_distances.tolist() + expected_predecessors = expected_predecessors.tolist() + + for i in range(num_expected_verts): + actual_distance = actual_distances[i] + expected_distance = expected_distances[actual_verts[i]] + # The distance value will be a MAX value (2**128) if there is no + # predecessor, so only do a closer compare if either the actual or + # expected are not that MAX value. + if (actual_distance <= 3.4e38) or (expected_distance <= 3.4e38): + assert actual_distance == \ + pytest.approx(expected_distance, 1e-4), \ + f"actual != expected for distance result at index {i}" + + # predecessors for graphs with multiple paths which are equally short + # are non-deterministic, so skip those checks for specific graph inputs. + # FIXME: add a helper to verify paths are correct when results are valid + # but non-deterministic + if ds_name not in ["karate.csv", "dolphins.csv"]: + assert actual_predecessors[i] == \ + pytest.approx(expected_predecessors[actual_verts[i]], 1e-4), \ + f"actual != expected for predecessor result at index {i}" From 76c1c7544324a62d460f67c16d76978de5d37ad3 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 25 Jan 2022 14:17:06 -0600 Subject: [PATCH 32/43] flake8 fixes. --- python/pylibcugraph/pylibcugraph/tests/conftest.py | 1 + python/pylibcugraph/pylibcugraph/tests/test_pagerank.py | 2 -- python/pylibcugraph/pylibcugraph/tests/test_sssp.py | 6 +++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/tests/conftest.py b/python/pylibcugraph/pylibcugraph/tests/conftest.py index 89d8fa57843..88d30f654af 100644 --- a/python/pylibcugraph/pylibcugraph/tests/conftest.py +++ b/python/pylibcugraph/pylibcugraph/tests/conftest.py @@ -133,6 +133,7 @@ def create_SGGraph(device_srcs, return (g, resource_handle) + # ============================================================================= # Pytest fixtures # ============================================================================= diff --git a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py index 76c88d53eb6..27b1731cc10 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_pagerank.py @@ -89,11 +89,9 @@ # auto-transposing in C) or raise the appropriate exception. def test_pagerank(sg_transposed_graph_objs): -#def test_pagerank(sg_graph_objs): from pylibcugraph.experimental import pagerank (g, resource_handle, ds_name) = sg_transposed_graph_objs - #(g, resource_handle, ds_name) = sg_graph_objs (expected_verts, expected_pageranks) = _test_data[ds_name] precomputed_vertex_out_weight_sums = None diff --git a/python/pylibcugraph/pylibcugraph/tests/test_sssp.py b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py index 8d7a9cd2a30..2d1fc83b08c 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_sssp.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py @@ -150,9 +150,9 @@ def test_sssp(sg_graph_objs): f"actual != expected for distance result at index {i}" # predecessors for graphs with multiple paths which are equally short - # are non-deterministic, so skip those checks for specific graph inputs. - # FIXME: add a helper to verify paths are correct when results are valid - # but non-deterministic + # are non-deterministic so skip those checks for specific graph inputs. + # FIXME: add a helper to verify paths are correct when results are + # valid but non-deterministic if ds_name not in ["karate.csv", "dolphins.csv"]: assert actual_predecessors[i] == \ pytest.approx(expected_predecessors[actual_verts[i]], 1e-4), \ From 1b1bce1258589dca597fc0469c874c404ad8e12d Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 25 Jan 2022 14:37:14 -0600 Subject: [PATCH 33/43] Clarified comment. --- python/pylibcugraph/pylibcugraph/tests/test_sssp.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_sssp.py b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py index 2d1fc83b08c..252fd57f3ed 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_sssp.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py @@ -149,8 +149,9 @@ def test_sssp(sg_graph_objs): pytest.approx(expected_distance, 1e-4), \ f"actual != expected for distance result at index {i}" - # predecessors for graphs with multiple paths which are equally short - # are non-deterministic so skip those checks for specific graph inputs. + # The array of predecessors for graphs with multiple paths that are + # equally short are non-deterministic, so skip those checks for specific + # graph inputs. # FIXME: add a helper to verify paths are correct when results are # valid but non-deterministic if ds_name not in ["karate.csv", "dolphins.csv"]: From 96aa3cc6b824a99901dac665a10ea24f14dc17dc Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 25 Jan 2022 22:18:33 -0600 Subject: [PATCH 34/43] Added documentation. --- python/pylibcugraph/pylibcugraph/README.md | 22 ++++ .../pylibcugraph/_cugraph_c/README.md | 5 + python/pylibcugraph/pylibcugraph/pagerank.pyx | 112 +++++++++++++----- python/pylibcugraph/pylibcugraph/sssp.pyx | 70 +++++++++-- 4 files changed, 170 insertions(+), 39 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/README.md b/python/pylibcugraph/pylibcugraph/README.md index e69de29bb2d..620b708d7a5 100644 --- a/python/pylibcugraph/pylibcugraph/README.md +++ b/python/pylibcugraph/pylibcugraph/README.md @@ -0,0 +1,22 @@ +# `pylibcugraph` + +This directory contains the sources to the `pylibcugraph` package. The sources +are primarily cython files which are built using the `setup.py` file in the +parent directory and depend on the `libcugraph_c` and `libcugraph` libraries and +headers. + +## components +The `connected_components` APIs. + +## structure +Internal utilities and types for use with the libcugraph C++ library. + +## utilities +Utility functions. + +## experimental +This subpackage defines the "experimental" APIs. many of these APIs are defined +elsewhere and simply imported into the `experimental/__init__.py` file. + +## tests +pytest tests for `pylibcugraph`. diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/README.md b/python/pylibcugraph/pylibcugraph/_cugraph_c/README.md index e69de29bb2d..cb0c6edff2b 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/README.md +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/README.md @@ -0,0 +1,5 @@ +# `pylibcugraph/_cugraph_c` + +This directory contains cython `.pxd` files which describe the cugraph C library +to cython. The contents here are simply a mapping of the cugraph_c C APIs to +cython for use in the cython code in the parent directory. \ No newline at end of file diff --git a/python/pylibcugraph/pylibcugraph/pagerank.pyx b/python/pylibcugraph/pylibcugraph/pagerank.pyx index ff8ad22a9b6..41426ecc708 100644 --- a/python/pylibcugraph/pylibcugraph/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/pagerank.pyx @@ -56,9 +56,6 @@ from pylibcugraph.utils cimport ( get_numpy_type_from_c_type, ) -import cupy -import numpy - def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, EXPERIMENTAL__Graph graph, @@ -69,18 +66,85 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, bool_t has_initial_guess, bool_t do_expensive_check): """ + Find the PageRank score for every vertex in a graph by computing an + approximation of the Pagerank eigenvector using the power method. The + number of iterations depends on the properties of the network itself; it + increases when the tolerance descreases and/or alpha increases toward the + limiting value of 1. + + Parameters + ---------- + handle : ResourceHandle + Handle to the underlying device resources needed for referencing data + and running algorithms. + + graph : Graph + The input graph. + + precomputed_vertex_out_weight_sums : None + This parameter is unsupported in this release and only None is + accepted. + + alpha : float + The damping factor alpha represents the probability to follow an + outgoing edge, standard value is 0.85. + Thus, 1.0-alpha is the probability to “teleport” to a random vertex. + Alpha should be greater than 0.0 and strictly lower than 1.0. + + epsilon : float + Set the tolerance the approximation, this parameter should be a small + magnitude value. + The lower the tolerance the better the approximation. If this value is + 0.0f, cuGraph will use the default value which is 1.0E-5. + Setting too small a tolerance can lead to non-convergence due to + numerical roundoff. Usually values between 0.01 and 0.00001 are + acceptable. + + max_iterations : int + The maximum number of iterations before an answer is returned. This can + be used to limit the execution time and do an early exit before the + solver reaches the convergence tolerance. + If this value is lower or equal to 0 cuGraph will use the default + value, which is 100. + + has_initial_guess : bool + + do_expensive_check : bool + + Returns + ------- + + Examples + -------- + """ + + # FIXME: import these modules here for now until a better pattern can be + # used for optional imports (perhaps 'import_optional()' from cugraph), or + # these are made hard dependencies. + try: + import cupy + except ModuleNotFoundError: + raise RuntimeError("pagerank requires the cupy package, which could " + "not be imported") + try: + import numpy + except ModuleNotFoundError: + raise RuntimeError("pagerank requires the numpy package, which could " + "not be imported") + assert_CAI_type(precomputed_vertex_out_weight_sums, "precomputed_vertex_out_weight_sums", allow_None=True) - cdef cugraph_resource_handle_t* c_resource_handle_ptr = resource_handle.c_resource_handle_ptr + cdef cugraph_resource_handle_t* c_resource_handle_ptr = \ + resource_handle.c_resource_handle_ptr cdef cugraph_graph_t* c_graph_ptr = graph.c_graph_ptr - cdef cugraph_type_erased_device_array_view_t* precomputed_vertex_out_weight_sums_ptr = NULL + cdef cugraph_type_erased_device_array_view_t* \ + precomputed_vertex_out_weight_sums_ptr = NULL if precomputed_vertex_out_weight_sums: - raise NotImplementedError("None is temporarily the only supported value for precomputed_vertex_out_weight_sums") - #precomputed_vertex_out_weight_sums_ptr = precomputed_vertex_out_weight_sums - pass + raise NotImplementedError("None is temporarily the only supported " + "value for precomputed_vertex_out_weight_sums") cdef cugraph_pagerank_result_t* result_ptr cdef cugraph_error_code_t error_code @@ -114,7 +178,13 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, num_vertices = cugraph_type_erased_device_array_view_size(vertices_ptr) - # Set up cupy arrays to return and copy results + # Set up cupy arrays to return and copy results to: + # * create cupy array object (these will be what the caller uses). + # * access the underlying device pointers. + # * create device array views which can be used with the copy APIs. + # * call copy APIs. This will copy data to the array pointed to the pointer + # in the cupy array objects that will be returned. + # * free view objects. cupy_vertices = cupy.array(numpy.zeros(num_vertices), dtype=vertex_numpy_type) cupy_pageranks = cupy.array(numpy.zeros(num_vertices), @@ -133,8 +203,6 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, cugraph_type_erased_device_array_view_create( cupy_pageranks_ptr, num_vertices, vertex_type) - # FIXME: free the views somewhere - error_code = cugraph_type_erased_device_array_view_copy( c_resource_handle_ptr, cupy_vertices_view_ptr, @@ -150,26 +218,8 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, assert_success(error_code, error_ptr, "cugraph_type_erased_device_array_view_copy") + cugraph_type_erased_device_array_view_free(cupy_pageranks_view_ptr) + cugraph_type_erased_device_array_view_free(cupy_vertices_view_ptr) cugraph_pagerank_result_free(result_ptr) return (cupy_vertices, cupy_pageranks) - - - -""" - cdef cugraph_pagerank_result_t* my_result - cugraph_pagerank(... , &my_result, ...) - - cdef cugraph_type_erased_device_array_t* verts - cdef cugraph_type_erased_device_array_t* prs - verts = cugraph_pagerank_result_get_vertices(my_result) - prs = cugraph_pagerank_result_get_pageranks(my_result) - - do device-device copy on verts to user array - do device-device copy on prs to user array -/* -size_t cugraph_type_erased_device_array_size(const cugraph_type_erased_device_array_t* p); -data_type_id_t cugraph_type_erased_device_array_type(const cugraph_type_erased_device_array_t* p); -const void* cugraph_type_erased_device_array_pointer(const cugraph_type_erased_device_array_t* p); -*/ -""" diff --git a/python/pylibcugraph/pylibcugraph/sssp.pyx b/python/pylibcugraph/pylibcugraph/sssp.pyx index f5f3b909935..01968b306ad 100644 --- a/python/pylibcugraph/pylibcugraph/sssp.pyx +++ b/python/pylibcugraph/pylibcugraph/sssp.pyx @@ -56,9 +56,6 @@ from pylibcugraph.utils cimport ( get_numpy_type_from_c_type, ) -import cupy -import numpy - def EXPERIMENTAL__sssp(EXPERIMENTAL__ResourceHandle resource_handle, EXPERIMENTAL__Graph graph, @@ -67,8 +64,58 @@ def EXPERIMENTAL__sssp(EXPERIMENTAL__ResourceHandle resource_handle, bool_t compute_predecessors, bool_t do_expensive_check): """ + Compute the distance and predecessors for shortest paths from the specified + source to all the vertices in the graph. The returned distances array will + contain the distance from the source to each vertex in the returned vertex + array at the same index. The returned predecessors array will contain the + previous vertex in the shortest path for each vertex in the vertex array at + the same index. Vertices that are unreachable will have a distance of + infinity denoted by the maximum value of the data type and the predecessor + set as -1. The source vertex predecessor will be set to -1. Graphs with + negative weight cycles are not supported. + + Parameters + ---------- + resource_handle : + + graph : + + source : + + cutoff : + + compute_predecessors : bool + + do_expensive_check : bool + + Returns + ------- + + Examples + -------- + """ - cdef cugraph_resource_handle_t* c_resource_handle_ptr = resource_handle.c_resource_handle_ptr + + # FIXME: import these modules here for now until a better pattern can be + # used for optional imports (perhaps 'import_optional()' from cugraph), or + # these are made hard dependencies. + try: + import cupy + except ModuleNotFoundError: + raise RuntimeError("sssp requires the cupy package, which could not " + "be imported") + try: + import numpy + except ModuleNotFoundError: + raise RuntimeError("sssp requires the numpy package, which could not " + "be imported") + + if compute_predecessors is False: + raise ValueError("compute_predecessors must be True for the current " + "release.") + + cdef cugraph_resource_handle_t* c_resource_handle_ptr = \ + resource_handle.c_resource_handle_ptr cdef cugraph_graph_t* c_graph_ptr = graph.c_graph_ptr cdef cugraph_paths_result_t* result_ptr @@ -106,7 +153,13 @@ def EXPERIMENTAL__sssp(EXPERIMENTAL__ResourceHandle resource_handle, num_vertices = cugraph_type_erased_device_array_view_size(vertices_ptr) - # Set up cupy arrays to return and copy results + # Set up cupy arrays to return and copy results to: + # * create cupy array object (these will be what the caller uses). + # * access the underlying device pointers. + # * create device array views which can be used with the copy APIs. + # * call copy APIs. This will copy data to the array pointed to the pointer + # in the cupy array objects that will be returned. + # * free view objects. cupy_vertices = cupy.array(numpy.zeros(num_vertices), dtype=vertex_numpy_type) cupy_distances = cupy.array(numpy.zeros(num_vertices), @@ -127,12 +180,10 @@ def EXPERIMENTAL__sssp(EXPERIMENTAL__ResourceHandle resource_handle, cdef cugraph_type_erased_device_array_view_t* cupy_distances_view_ptr = \ cugraph_type_erased_device_array_view_create( cupy_distances_ptr, num_vertices, vertex_type) - cdef cugraph_type_erased_device_array_view_t* cupy_predecessors_view_ptr = \ + cdef cugraph_type_erased_device_array_view_t* cupy_predecessors_view_ptr =\ cugraph_type_erased_device_array_view_create( cupy_predecessors_ptr, num_vertices, vertex_type) - # FIXME: free the views somewhere - error_code = cugraph_type_erased_device_array_view_copy( c_resource_handle_ptr, cupy_vertices_view_ptr, @@ -155,6 +206,9 @@ def EXPERIMENTAL__sssp(EXPERIMENTAL__ResourceHandle resource_handle, assert_success(error_code, error_ptr, "cugraph_type_erased_device_array_view_copy") + cugraph_type_erased_device_array_view_free(cupy_distances_view_ptr) + cugraph_type_erased_device_array_view_free(cupy_predecessors_view_ptr) + cugraph_type_erased_device_array_view_free(cupy_vertices_view_ptr) cugraph_paths_result_free(result_ptr) return (cupy_vertices, cupy_distances, cupy_predecessors) From 0824dca8be1441670c65dc452b587991af7d5dc7 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Wed, 26 Jan 2022 00:05:14 -0800 Subject: [PATCH 35/43] bug fix --- cpp/src/structure/graph_impl.cuh | 2 +- cpp/tests/structure/transpose_storage_test.cpp | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cpp/src/structure/graph_impl.cuh b/cpp/src/structure/graph_impl.cuh index e969bb4a6a3..ef64e60ac2f 100644 --- a/cpp/src/structure/graph_impl.cuh +++ b/cpp/src/structure/graph_impl.cuh @@ -1190,8 +1190,8 @@ graph_t( handle, std::move(vertex_span), - std::move(edgelist_cols), std::move(edgelist_rows), + std::move(edgelist_cols), std::move(edgelist_weights), graph_properties_t{is_multigraph, false}, renumber); diff --git a/cpp/tests/structure/transpose_storage_test.cpp b/cpp/tests/structure/transpose_storage_test.cpp index c216d8d89c4..a44d50f6f49 100644 --- a/cpp/tests/structure/transpose_storage_test.cpp +++ b/cpp/tests/structure/transpose_storage_test.cpp @@ -149,10 +149,9 @@ class Tests_TransposeStorage std::vector> storage_transposed_edges( h_storage_transposed_rows.size()); for (size_t i = 0; i < storage_transposed_edges.size(); ++i) { - storage_transposed_edges[i] = - std::make_tuple(h_storage_transposed_cols[i], - h_storage_transposed_rows[i], - (*h_storage_transposed_weights)[i]); // flip rows and cols + storage_transposed_edges[i] = std::make_tuple(h_storage_transposed_rows[i], + h_storage_transposed_cols[i], + (*h_storage_transposed_weights)[i]); } std::sort(storage_transposed_edges.begin(), storage_transposed_edges.end()); @@ -168,8 +167,8 @@ class Tests_TransposeStorage std::vector> storage_transposed_edges( h_storage_transposed_rows.size()); for (size_t i = 0; i < storage_transposed_edges.size(); ++i) { - storage_transposed_edges[i] = std::make_tuple( - h_storage_transposed_cols[i], h_storage_transposed_rows[i]); // flip rows and cols + storage_transposed_edges[i] = + std::make_tuple(h_storage_transposed_rows[i], h_storage_transposed_cols[i]); } std::sort(storage_transposed_edges.begin(), storage_transposed_edges.end()); From 48b5c511a3696c690125d47ee31e093fb83ffc55 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Wed, 26 Jan 2022 09:19:09 -0600 Subject: [PATCH 36/43] Added docstring content, added checks to ensure unsupported params are set correctly, fixed flake8 error. --- python/pylibcugraph/pylibcugraph/graphs.pyx | 40 ++++++++++++++++++ python/pylibcugraph/pylibcugraph/pagerank.pyx | 39 ++++++++++++++++-- python/pylibcugraph/pylibcugraph/sssp.pyx | 41 +++++++++++++++++-- .../pylibcugraph/tests/test_sssp.py | 4 +- 4 files changed, 115 insertions(+), 9 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/graphs.pyx b/python/pylibcugraph/pylibcugraph/graphs.pyx index 0f0494d3919..f866de22da7 100644 --- a/python/pylibcugraph/pylibcugraph/graphs.pyx +++ b/python/pylibcugraph/pylibcugraph/graphs.pyx @@ -52,6 +52,46 @@ cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): """ RAII-stye Graph class for use with single-GPU APIs that manages the individual create/free calls and the corresponding cugraph_graph_t pointer. + + Parameters + ---------- + resource_handle : ResourceHandle + Handle to the underlying device resources needed for referencing data + and running algorithms. + + graph_properties : GraphProperties + Object defining intended properties for the graph. + + src_array : device array type + Device array containing the vertex identifiers of the source of each + directed edge. The order of the array corresponds to the ordering of the + dst_array, where the ith item in src_array and the ith item in dst_array + define the ith edge of the graph. + + dst_array : device array type + Device array containing the vertex identifiers of the destination of each + directed edge. The order of the array corresponds to the ordering of the + src_array, where the ith item in src_array and the ith item in dst_array + define the ith edge of the graph. + + weight_array : device array type + Device array containing the weight values of each directed edge. The + order of the array corresponds to the ordering of the src_array and + dst_array arrays, where the ith item in weight_array is the weight value + of the ith edge of the graph. + + store_transposed : bool + Set to True if the graph should be transposed. This is required for some + algorithms, such as pagerank. + + renumber : bool + Set to True to indicate the vertices used in src_array and dst_array are + not appropriate for use as internal array indices, and should be mapped + to continuous integrers starting from 0. + + do_expensive_check : bool + If True, performs more extensive tests on the inputs to ensure + validitity, at the expense of increased run time. """ def __cinit__(self, EXPERIMENTAL__ResourceHandle resource_handle, diff --git a/python/pylibcugraph/pylibcugraph/pagerank.pyx b/python/pylibcugraph/pylibcugraph/pagerank.pyx index 41426ecc708..6d6bbf5c399 100644 --- a/python/pylibcugraph/pylibcugraph/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/pagerank.pyx @@ -74,12 +74,13 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, Parameters ---------- - handle : ResourceHandle + resource_handle : ResourceHandle Handle to the underlying device resources needed for referencing data and running algorithms. - graph : Graph - The input graph. + graph : SGGraph + The input graph. The graph must be created with the store_transposed + option set to True. precomputed_vertex_out_weight_sums : None This parameter is unsupported in this release and only None is @@ -108,15 +109,41 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, value, which is 100. has_initial_guess : bool + This parameter is unsupported in this release and only False is + accepted. do_expensive_check : bool + If True, performs more extensive tests on the inputs to ensure + validitity, at the expense of increased run time. Returns ------- + A tuple of device arrays, where the first item in the tuple is a device + array containing the vertex identifiers, and the second item is a device + array containing the pagerank values for the corresponding vertices. For + example, the vertex identifier at the ith element of the vertex array has + the pagerank value of the ith element in the pagerank array. Examples -------- - + >>> import pylibcugraph, cupy, numpy + >>> srcs = cupy.asarray([0, 1, 2], dtype=numpy.int32) + >>> dsts = cupy.asarray([1, 2, 3], dtype=numpy.int32) + >>> weights = cupy.asarray([1.0, 1.0, 1.0], dtype=numpy.float32) + >>> resource_handle = pylibcugraph.experimental.ResourceHandle() + >>> graph_props = pylibcugraph.experimental.GraphProperties( + ... is_symmetric=False, is_multigraph=False) + >>> G = pylibcugraph.experimental.SGGraph( + ... resource_handle, graph_props, srcs, dsts, weights, + ... store_transposed=True, renumber=False, expensive_check=False) + >>> (vertices, pageranks) = pylibcugraph.experimental.pagerank( + ... resource_handle, G, None, alpha=0.85, epsilon=1.0e-6, + ... max_iterations=500, has_initial_guess=False, + ... do_expensive_check=False) + >>> vertices + array([0, 1, 2, 3], dtype=int32) + >>> pageranks + array([0.11615585, 0.21488841, 0.2988108 , 0.3701449 ], dtype=float32) """ # FIXME: import these modules here for now until a better pattern can be @@ -133,6 +160,10 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, raise RuntimeError("pagerank requires the numpy package, which could " "not be imported") + if has_initial_guess is True: + raise ValueError("has_initial_guess must be False for the current " + "release.") + assert_CAI_type(precomputed_vertex_out_weight_sums, "precomputed_vertex_out_weight_sums", allow_None=True) diff --git a/python/pylibcugraph/pylibcugraph/sssp.pyx b/python/pylibcugraph/pylibcugraph/sssp.pyx index 01968b306ad..3c668b82f66 100644 --- a/python/pylibcugraph/pylibcugraph/sssp.pyx +++ b/python/pylibcugraph/pylibcugraph/sssp.pyx @@ -76,24 +76,59 @@ def EXPERIMENTAL__sssp(EXPERIMENTAL__ResourceHandle resource_handle, Parameters ---------- - resource_handle : + resource_handle : ResourceHandle + Handle to the underlying device resources needed for referencing data + and running algorithms. - graph : + graph : SGGraph + The input graph. source : + The vertex identifier of the source vertex. cutoff : + Maximum edge weight sum to consider. compute_predecessors : bool + This parameter must be set to True for this release. do_expensive_check : bool + If True, performs more extensive tests on the inputs to ensure + validitity, at the expense of increased run time. Returns ------- + A 3-tuple, where the first item in the tuple is a device array containing + the vertex identifiers, the second item is a device array containing the + distance for each vertex from the source vertex, and the third item is a + device array containing the vertex identifier of the preceding vertex in the + path for that vertex. For example, the vertex identifier at the ith element + of the vertex array has a distance from the source vertex of the ith element + in the distance array, and the preceding vertex in the path is the ith + element in the predecessor array. Examples -------- - + >>> import pylibcugraph, cupy, numpy + >>> srcs = cupy.asarray([0, 1, 2], dtype=numpy.int32) + >>> dsts = cupy.asarray([1, 2, 3], dtype=numpy.int32) + >>> weights = cupy.asarray([1.0, 1.0, 1.0], dtype=numpy.float32) + >>> resource_handle = pylibcugraph.experimental.ResourceHandle() + >>> graph_props = pylibcugraph.experimental.GraphProperties( + ... is_symmetric=False, is_multigraph=False) + >>> G = pylibcugraph.experimental.SGGraph( + ... resource_handle, graph_props, srcs, dsts, weights, + ... store_transposed=False, renumber=False, expensive_check=False) + >>> (vertices, distances, predecessors) = pylibcugraph.experimental.sssp( + ... resource_handle, G, source=1, cutoff=999, + ... compute_predecessors=True, do_expensive_check=False) + >>> vertices + array([0, 1, 2, 3], dtype=int32) + >>> distances + array([3.4028235e+38, 0.0000000e+00, 1.0000000e+00, 2.0000000e+00], + dtype=float32) + >>> predecessors + array([-1, -1, 1, 2], dtype=int32) """ # FIXME: import these modules here for now until a better pattern can be diff --git a/python/pylibcugraph/pylibcugraph/tests/test_sssp.py b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py index 252fd57f3ed..9649a9f98df 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_sssp.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_sssp.py @@ -150,8 +150,8 @@ def test_sssp(sg_graph_objs): f"actual != expected for distance result at index {i}" # The array of predecessors for graphs with multiple paths that are - # equally short are non-deterministic, so skip those checks for specific - # graph inputs. + # equally short are non-deterministic, so skip those checks for + # specific graph inputs. # FIXME: add a helper to verify paths are correct when results are # valid but non-deterministic if ds_name not in ["karate.csv", "dolphins.csv"]: From cf5b25e4f2a9abb74da1c29ac860d65221516a80 Mon Sep 17 00:00:00 2001 From: Chuck Hastings Date: Wed, 26 Jan 2022 12:31:13 -0500 Subject: [PATCH 37/43] update tests to reflect bug fix for transpose --- cpp/tests/c_api/bfs_test.c | 6 ++--- cpp/tests/c_api/extract_paths_test.c | 6 ++--- cpp/tests/c_api/pagerank_test.c | 40 +++++++++++++++++++++++++++- cpp/tests/c_api/sssp_test.c | 6 ++--- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/cpp/tests/c_api/bfs_test.c b/cpp/tests/c_api/bfs_test.c index 63ca2943797..7061a59f237 100644 --- a/cpp/tests/c_api/bfs_test.c +++ b/cpp/tests/c_api/bfs_test.c @@ -145,9 +145,9 @@ int test_bfs_with_transpose() vertex_t src[] = {0, 1, 1, 2, 2, 2, 3, 4}; vertex_t dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; weight_t wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; - vertex_t seeds[] = {5}; - vertex_t expected_distances[] = {3, 2, 2, 1, 1, 0}; - vertex_t expected_predecessors[] = {1, 3, 3, 5, 5, -1}; + vertex_t seeds[] = {0}; + vertex_t expected_distances[] = {0, 1, 2147483647, 2, 2, 3}; + vertex_t expected_predecessors[] = {-1, 0, -1, 1, 1, 3}; // Bfs wants store_transposed = FALSE // This call will force cugraph_bfs to transpose the graph diff --git a/cpp/tests/c_api/extract_paths_test.c b/cpp/tests/c_api/extract_paths_test.c index 919b3d830ed..3f9ea8e3911 100644 --- a/cpp/tests/c_api/extract_paths_test.c +++ b/cpp/tests/c_api/extract_paths_test.c @@ -168,10 +168,10 @@ int test_bfs_with_extract_paths_with_transpose() vertex_t src[] = {0, 1, 1, 2, 2, 2, 3, 4}; vertex_t dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; weight_t wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; - vertex_t seeds[] = {5}; - vertex_t destinations[] = {0}; + vertex_t seeds[] = {0}; + vertex_t destinations[] = {5}; vertex_t expected_max_path_length = 4; - vertex_t expected_paths[] = {5, 3, 1, 0}; + vertex_t expected_paths[] = {0, 1, 3, 5}; // Bfs wants store_transposed = FALSE // This call will force cugraph_bfs to transpose the graph diff --git a/cpp/tests/c_api/pagerank_test.c b/cpp/tests/c_api/pagerank_test.c index 144c26244c6..f8a9e71afb0 100644 --- a/cpp/tests/c_api/pagerank_test.c +++ b/cpp/tests/c_api/pagerank_test.c @@ -128,7 +128,43 @@ int test_pagerank_with_transpose() // This call will force cugraph_pagerank to transpose the graph // But we're passing src/dst backwards so the results will be the same return generic_pagerank_test( - h_dst, h_src, h_wgt, h_result, num_vertices, num_edges, FALSE, alpha, epsilon, max_iterations); + h_src, h_dst, h_wgt, h_result, num_vertices, num_edges, FALSE, alpha, epsilon, max_iterations); +} + +int test_pagerank_4() +{ + size_t num_edges = 3; + size_t num_vertices = 4; + + vertex_t h_src[] = {0, 1, 2}; + vertex_t h_dst[] = {1, 2, 3}; + weight_t h_wgt[] = {1.f, 1.f, 1.f}; + weight_t h_result[] = {0.11615584790706635f, 0.21488840878009796f, 0.29881080985069275f, 0.37014490365982056f}; + + double alpha = 0.85; + double epsilon = 1.0e-6; + size_t max_iterations = 500; + + return generic_pagerank_test( + h_src, h_dst, h_wgt, h_result, num_vertices, num_edges, FALSE, alpha, epsilon, max_iterations); +} + +int test_pagerank_4_with_transpose() +{ + size_t num_edges = 3; + size_t num_vertices = 4; + + vertex_t h_src[] = {0, 1, 2}; + vertex_t h_dst[] = {1, 2, 3}; + weight_t h_wgt[] = {1.f, 1.f, 1.f}; + weight_t h_result[] = {0.11615584790706635f, 0.21488840878009796f, 0.29881080985069275f, 0.37014490365982056f}; + + double alpha = 0.85; + double epsilon = 1.0e-6; + size_t max_iterations = 500; + + return generic_pagerank_test( + h_src, h_dst, h_wgt, h_result, num_vertices, num_edges, TRUE, alpha, epsilon, max_iterations); } /******************************************************************************/ @@ -138,5 +174,7 @@ int main(int argc, char** argv) int result = 0; result |= RUN_TEST(test_pagerank); result |= RUN_TEST(test_pagerank_with_transpose); + result |= RUN_TEST(test_pagerank_4); + result |= RUN_TEST(test_pagerank_4_with_transpose); return result; } diff --git a/cpp/tests/c_api/sssp_test.c b/cpp/tests/c_api/sssp_test.c index 4a906162ce0..4b210d3bcc7 100644 --- a/cpp/tests/c_api/sssp_test.c +++ b/cpp/tests/c_api/sssp_test.c @@ -135,15 +135,15 @@ int test_sssp_with_transpose() vertex_t src[] = {0, 1, 1, 2, 2, 2, 3, 4}; vertex_t dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; weight_t wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; - weight_t expected_distances[] = {4.4, 4.3, 7.4, 7.2f, 3.2f, 0.0f}; - vertex_t expected_predecessors[] = {1, 4, 1, 5, 5, -1}; + weight_t expected_distances[] = {0.0f, 0.1f, FLT_MAX, 2.2f, 1.2f, 4.4f}; + vertex_t expected_predecessors[] = {-1, 0, -1, 1, 1, 4}; // Bfs wants store_transposed = FALSE // This call will force cugraph_sssp to transpose the graph return generic_sssp_test(src, dst, wgt, - 5, + 0, expected_distances, expected_predecessors, num_vertices, From 6748668ec8071f8e1c2acf2317396041a721ba2e Mon Sep 17 00:00:00 2001 From: Chuck Hastings Date: Wed, 26 Jan 2022 12:36:48 -0500 Subject: [PATCH 38/43] add FIXME to code that is current ifdeffed out --- cpp/include/cugraph_c/array.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/include/cugraph_c/array.h b/cpp/include/cugraph_c/array.h index 1f24889800a..86ed6d9208b 100644 --- a/cpp/include/cugraph_c/array.h +++ b/cpp/include/cugraph_c/array.h @@ -64,6 +64,8 @@ cugraph_error_code_t cugraph_type_erased_device_array_create( void cugraph_type_erased_device_array_free(cugraph_type_erased_device_array_t* p); #if 0 +// FIXME: Not implemented, need to discuss if this can work. We will either implement +// this later or delete it from the interface once we resolve how to handle this /** * @brief Release the raw pointer of the type erased device array * @@ -155,6 +157,8 @@ cugraph_error_code_t cugraph_type_erased_host_array_create(const cugraph_resourc void cugraph_type_erased_host_array_free(cugraph_type_erased_host_array_t* p); #if 0 +// FIXME: Not implemented, need to discuss if this can work. We will either implement +// this later or delete it from the interface once we resolve how to handle this /** * @brief Release the raw pointer of the type erased host array * From d2b91761c8a6ee9f11bd044e6d95eba9a671334f Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Wed, 26 Jan 2022 14:01:46 -0600 Subject: [PATCH 39/43] Added FIXMEs based on review feedback, fixed typo in docstring. --- python/pylibcugraph/pylibcugraph/graphs.pyx | 4 +++- python/pylibcugraph/pylibcugraph/pagerank.pyx | 1 + python/pylibcugraph/pylibcugraph/utils.pyx | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/python/pylibcugraph/pylibcugraph/graphs.pyx b/python/pylibcugraph/pylibcugraph/graphs.pyx index f866de22da7..82e10c4e187 100644 --- a/python/pylibcugraph/pylibcugraph/graphs.pyx +++ b/python/pylibcugraph/pylibcugraph/graphs.pyx @@ -87,7 +87,7 @@ cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): renumber : bool Set to True to indicate the vertices used in src_array and dst_array are not appropriate for use as internal array indices, and should be mapped - to continuous integrers starting from 0. + to continuous integers starting from 0. do_expensive_check : bool If True, performs more extensive tests on the inputs to ensure @@ -117,6 +117,8 @@ cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): assert_CAI_type(dst_array, "dst_array") assert_CAI_type(weight_array, "weight_array") + # FIXME: assert that src_array and dst_array have the same type + cdef cugraph_error_t* error_ptr cdef cugraph_error_code_t error_code diff --git a/python/pylibcugraph/pylibcugraph/pagerank.pyx b/python/pylibcugraph/pylibcugraph/pagerank.pyx index 6d6bbf5c399..b26f6ff84a1 100644 --- a/python/pylibcugraph/pylibcugraph/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/pagerank.pyx @@ -167,6 +167,7 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, assert_CAI_type(precomputed_vertex_out_weight_sums, "precomputed_vertex_out_weight_sums", allow_None=True) + # FIXME: assert that precomputed_vertex_out_weight_sums type == weight type cdef cugraph_resource_handle_t* c_resource_handle_ptr = \ resource_handle.c_resource_handle_ptr diff --git a/python/pylibcugraph/pylibcugraph/utils.pyx b/python/pylibcugraph/pylibcugraph/utils.pyx index dc154e3fa85..608773a6e9a 100644 --- a/python/pylibcugraph/pylibcugraph/utils.pyx +++ b/python/pylibcugraph/pylibcugraph/utils.pyx @@ -36,6 +36,7 @@ cdef assert_success(cugraph_error_code_t code, else: code_str = "unknown error code" # FIXME: extract message using cugraph_error_message() + # FIXME: If error_ptr has a value, free it using cugraph_error_free() raise RuntimeError(f"non-success value returned from {api_name}: {code_str}") From 60549ba870b455f7d3a296fefec0ad65f89d8f9d Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Wed, 26 Jan 2022 14:14:49 -0600 Subject: [PATCH 40/43] Fixed inconsistency in param name/doc string, added FIXME for more test coverage. --- python/pylibcugraph/pylibcugraph/graphs.pyx | 10 +++++----- python/pylibcugraph/pylibcugraph/pagerank.pyx | 3 +-- python/pylibcugraph/pylibcugraph/tests/conftest.py | 2 ++ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/graphs.pyx b/python/pylibcugraph/pylibcugraph/graphs.pyx index 82e10c4e187..381191c3e51 100644 --- a/python/pylibcugraph/pylibcugraph/graphs.pyx +++ b/python/pylibcugraph/pylibcugraph/graphs.pyx @@ -101,7 +101,7 @@ cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): weight_array, store_transposed, renumber, - expensive_check): + do_expensive_check): # FIXME: add tests for these if not(isinstance(store_transposed, (int, bool))): @@ -110,9 +110,9 @@ cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): if not(isinstance(renumber, (int, bool))): raise TypeError("expected int or bool for renumber, got " f"{type(renumber)}") - if not(isinstance(expensive_check, (int, bool))): - raise TypeError("expected int or bool for expensive_check, got " - f"{type(expensive_check)}") + if not(isinstance(do_expensive_check, (int, bool))): + raise TypeError("expected int or bool for do_expensive_check, got " + f"{type(do_expensive_check)}") assert_CAI_type(src_array, "src_array") assert_CAI_type(dst_array, "dst_array") assert_CAI_type(weight_array, "weight_array") @@ -157,7 +157,7 @@ cdef class EXPERIMENTAL__SGGraph(EXPERIMENTAL__Graph): weights_view_ptr, store_transposed, renumber, - expensive_check, + do_expensive_check, &(self.c_graph_ptr), &error_ptr) diff --git a/python/pylibcugraph/pylibcugraph/pagerank.pyx b/python/pylibcugraph/pylibcugraph/pagerank.pyx index b26f6ff84a1..1483b0cdc35 100644 --- a/python/pylibcugraph/pylibcugraph/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/pagerank.pyx @@ -79,8 +79,7 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, and running algorithms. graph : SGGraph - The input graph. The graph must be created with the store_transposed - option set to True. + The input graph. precomputed_vertex_out_weight_sums : None This parameter is unsupported in this release and only None is diff --git a/python/pylibcugraph/pylibcugraph/tests/conftest.py b/python/pylibcugraph/pylibcugraph/tests/conftest.py index 88d30f654af..132e0958c64 100644 --- a/python/pylibcugraph/pylibcugraph/tests/conftest.py +++ b/python/pylibcugraph/pylibcugraph/tests/conftest.py @@ -131,6 +131,8 @@ def create_SGGraph(device_srcs, renumber=False, expensive_check=False) + # FIXME: add coverage for renumber=True and expensive_check=True + return (g, resource_handle) From 404f3fcd64622061b51a0b5bf72b274eba5a2490 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Wed, 26 Jan 2022 14:27:33 -0600 Subject: [PATCH 41/43] Added FIXMEs --- cpp/src/c_api/graph_mg.cpp | 6 ++++++ cpp/src/c_api/graph_sg.cpp | 3 +++ cpp/tests/c_api/bfs_test.c | 5 +++++ python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd | 2 ++ 4 files changed, 16 insertions(+) diff --git a/cpp/src/c_api/graph_mg.cpp b/cpp/src/c_api/graph_mg.cpp index b8027bc1b37..b2448e46796 100644 --- a/cpp/src/c_api/graph_mg.cpp +++ b/cpp/src/c_api/graph_mg.cpp @@ -16,6 +16,12 @@ #include +// FIXME: assume that this function will directly call +// create_graph_from_edgelist() instead of invoking a graph constructor, in +// which case, some parameters here (e.g. vertex_partition_offsets, +// segment_offsets) are related to implementation details and unnecessary if +// this function calls create_graph_from_edgelist(). + extern "C" cugraph_error_code_t cugraph_mg_graph_create( const cugraph_resource_handle_t* handle, const cugraph_graph_properties_t* properties, diff --git a/cpp/src/c_api/graph_sg.cpp b/cpp/src/c_api/graph_sg.cpp index bf94003e9f5..6546e135bce 100644 --- a/cpp/src/c_api/graph_sg.cpp +++ b/cpp/src/c_api/graph_sg.cpp @@ -185,6 +185,9 @@ extern "C" cugraph_error_code_t cugraph_sg_graph_create( cugraph_error_t** error) { constexpr bool multi_gpu = false; + // FIXME: There could be a case where the vertex_t is int64_t but the number + // of edges is less than 2^31-1. The if statement below could then be modified + // to catch this case. constexpr size_t int32_threshold{std::numeric_limits::max()}; *graph = nullptr; diff --git a/cpp/tests/c_api/bfs_test.c b/cpp/tests/c_api/bfs_test.c index 7061a59f237..e62237cc0cd 100644 --- a/cpp/tests/c_api/bfs_test.c +++ b/cpp/tests/c_api/bfs_test.c @@ -54,6 +54,11 @@ int generic_bfs_test(vertex_t* h_src, ret_code = create_test_graph( p_handle, h_src, h_dst, h_wgt, num_edges, store_transposed, &p_graph, &ret_error); + /* + * FIXME: in create_graph_test.c, variables are defined but then hard-coded to + * the constant INT32. It would be better to pass the types into the functions + * in both cases so that the test cases could be parameterized in the main. + */ ret_code = cugraph_type_erased_device_array_create(p_handle, num_seeds, INT32, &p_sources, &ret_error); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "p_sources create failed."); diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd index 511b4a19048..8cc4b06093e 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/graph.pxd @@ -56,6 +56,8 @@ cdef extern from "cugraph_c/graph.h": cugraph_graph_t* graph ) + # FIXME: the API parameters may change to remove vertex_partition_offsets + # and segment_offsets cdef cugraph_error_code_t \ cugraph_mg_graph_create( const cugraph_resource_handle_t* handle, From f761712128d42548fec2d842c02fce48d25eb889 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Wed, 26 Jan 2022 14:30:18 -0600 Subject: [PATCH 42/43] Fixed param name after changing it in the API. --- python/pylibcugraph/pylibcugraph/tests/conftest.py | 4 ++-- python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/tests/conftest.py b/python/pylibcugraph/pylibcugraph/tests/conftest.py index 132e0958c64..df77e58d6be 100644 --- a/python/pylibcugraph/pylibcugraph/tests/conftest.py +++ b/python/pylibcugraph/pylibcugraph/tests/conftest.py @@ -129,9 +129,9 @@ def create_SGGraph(device_srcs, device_weights, store_transposed=transposed, renumber=False, - expensive_check=False) + do_expensive_check=False) - # FIXME: add coverage for renumber=True and expensive_check=True + # FIXME: add coverage for renumber=True and do_expensive_check=True return (g, resource_handle) diff --git a/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py b/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py index 73a2a984a7a..fedcca8cae8 100644 --- a/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py +++ b/python/pylibcugraph/pylibcugraph/tests/test_graph_sg.py @@ -88,7 +88,7 @@ def test_sg_graph(graph_data): device_weights, store_transposed=False, renumber=False, - expensive_check=False) + do_expensive_check=False) # call SGGraph.__dealloc__() del g @@ -101,4 +101,4 @@ def test_sg_graph(graph_data): device_weights, store_transposed=False, renumber=False, - expensive_check=False) + do_expensive_check=False) From 22667c6ee90bba8f3b1c4db1b0c815a8f7028489 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Wed, 26 Jan 2022 14:57:20 -0600 Subject: [PATCH 43/43] Minor update to docstring example to match param update. --- python/pylibcugraph/pylibcugraph/pagerank.pyx | 2 +- python/pylibcugraph/pylibcugraph/sssp.pyx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/pagerank.pyx b/python/pylibcugraph/pylibcugraph/pagerank.pyx index 1483b0cdc35..a1b5a704693 100644 --- a/python/pylibcugraph/pylibcugraph/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/pagerank.pyx @@ -134,7 +134,7 @@ def EXPERIMENTAL__pagerank(EXPERIMENTAL__ResourceHandle resource_handle, ... is_symmetric=False, is_multigraph=False) >>> G = pylibcugraph.experimental.SGGraph( ... resource_handle, graph_props, srcs, dsts, weights, - ... store_transposed=True, renumber=False, expensive_check=False) + ... store_transposed=True, renumber=False, do_expensive_check=False) >>> (vertices, pageranks) = pylibcugraph.experimental.pagerank( ... resource_handle, G, None, alpha=0.85, epsilon=1.0e-6, ... max_iterations=500, has_initial_guess=False, diff --git a/python/pylibcugraph/pylibcugraph/sssp.pyx b/python/pylibcugraph/pylibcugraph/sssp.pyx index 3c668b82f66..af3eed36186 100644 --- a/python/pylibcugraph/pylibcugraph/sssp.pyx +++ b/python/pylibcugraph/pylibcugraph/sssp.pyx @@ -118,7 +118,7 @@ def EXPERIMENTAL__sssp(EXPERIMENTAL__ResourceHandle resource_handle, ... is_symmetric=False, is_multigraph=False) >>> G = pylibcugraph.experimental.SGGraph( ... resource_handle, graph_props, srcs, dsts, weights, - ... store_transposed=False, renumber=False, expensive_check=False) + ... store_transposed=False, renumber=False, do_expensive_check=False) >>> (vertices, distances, predecessors) = pylibcugraph.experimental.sssp( ... resource_handle, G, source=1, cutoff=999, ... compute_predecessors=True, do_expensive_check=False)