Skip to content

Commit

Permalink
wraps if with stream params into detail ns
Browse files Browse the repository at this point in the history
  • Loading branch information
elstehle committed Jul 16, 2022
1 parent 4aaf595 commit 01aef44
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 20 deletions.
12 changes: 4 additions & 8 deletions cpp/src/io/json/nested_json.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,7 @@

#include <rmm/cuda_stream_view.hpp>

namespace cudf {
namespace io {
namespace json {
namespace gpu {
namespace cudf::io::json::gpu {

/// Type used to represent the atomic symbol type used within the finite-state machine
using SymbolT = char;
Expand Down Expand Up @@ -79,6 +76,7 @@ enum token_t : PdaTokenT {
NUM_TOKENS
};

namespace detail {
/**
* @brief Identifies the stack context for each character from a JSON input. Specifically, we
* identify brackets and braces outside of quoted fields (e.g., field names, strings).
Expand Down Expand Up @@ -109,8 +107,6 @@ void get_token_stream(device_span<SymbolT const> d_json_in,
device_span<SymbolOffsetT> d_tokens_indices,
SymbolOffsetT* d_num_written_tokens,
rmm::cuda_stream_view stream);
} // namespace detail

} // namespace gpu
} // namespace json
} // namespace io
} // namespace cudf
} // namespace cudf::io::json::gpu
14 changes: 6 additions & 8 deletions cpp/src/io/json/nested_json_gpu.cu
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,7 @@

#include <rmm/exec_policy.hpp>

namespace cudf {
namespace io {
namespace json {
namespace gpu {
namespace cudf::io::json::gpu {

//------------------------------------------------------------------------------
// JSON-TO-STACK-OP DFA
Expand Down Expand Up @@ -320,6 +317,8 @@ struct JSONToStackOp {
}
};

namespace detail {

void get_stack_context(device_span<SymbolT const> d_json_in,
device_span<SymbolT> d_top_of_stack,
rmm::cuda_stream_view stream)
Expand Down Expand Up @@ -410,7 +409,6 @@ void get_token_stream(device_span<SymbolT const> d_json_in,
stream);
}

} // namespace gpu
} // namespace json
} // namespace io
} // namespace cudf
} // namespace detail

} // namespace cudf::io::json::gpu
7 changes: 3 additions & 4 deletions cpp/tests/io/nested_json_test.cu
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ TEST_F(JsonTest, StackContext)
d_input.data(), input.data(), input.size() * sizeof(SymbolT), cudaMemcpyHostToDevice, stream));

// Run algorithm
cudf::io::json::gpu::get_stack_context(
cudf::io::json::gpu::detail::get_stack_context(
d_input,
cudf::device_span<StackSymbolT>{stack_context.device_ptr(), stack_context.size()},
stream);
Expand Down Expand Up @@ -130,13 +130,12 @@ TEST_F(JsonTest, TokenStream)
ASSERT_CUDA_SUCCEEDED(cudaMemcpyAsync(
d_input.data(), input.data(), input.size() * sizeof(SymbolT), cudaMemcpyHostToDevice, stream));


hostdevice_vector<PdaTokenT> tokens_gpu{input.size(), stream};
hostdevice_vector<SymbolOffsetT> token_indices_gpu{input.size(), stream};
hostdevice_vector<SymbolOffsetT> num_tokens_out{single_item, stream};

// Parse the JSON and get the token stream
cudf::io::json::gpu::get_token_stream(
cudf::io::json::gpu::detail::get_token_stream(
d_input,
cudf::device_span<PdaTokenT>{tokens_gpu.device_ptr(), tokens_gpu.size()},
cudf::device_span<SymbolOffsetT>{token_indices_gpu.device_ptr(), token_indices_gpu.size()},
Expand Down Expand Up @@ -179,7 +178,7 @@ TEST_F(JsonTest, TokenStream)

// Verify the number of tokens matches
ASSERT_EQ(golden_token_stream.size(), num_tokens_out[0]);

for (std::size_t i = 0; i < num_tokens_out[0]; i++) {
// Ensure the index the tokens are pointing to do match
ASSERT_EQ(golden_token_stream[i].first, token_indices_gpu[i]);
Expand Down

0 comments on commit 01aef44

Please sign in to comment.