From 082e13f86f3ec37a3a751419ddfcf53193b10d8b Mon Sep 17 00:00:00 2001 From: Bruno Tremblay Date: Fri, 1 Dec 2023 15:00:42 -0500 Subject: [PATCH] adapt to bigrquery dev version > 1.4.2 --- DESCRIPTION | 4 +- NAMESPACE | 7 ++- NEWS.md | 3 ++ R/RcppExports.R | 10 ++--- R/bigrquerystorage.R | 10 +++-- R/bqs_download.R | 74 +++++++++++++++++++++++++------ R/client_info.R | 2 +- R/dbi.R | 2 +- man/bigrquerystorage-package.Rd | 26 +++++++++++ man/bqs_table_download.Rd | 11 ++--- man/bqs_ua.Rd | 11 ----- src/Makevars.in | 2 +- tests/testthat/test-integration.R | 8 ++-- 13 files changed, 125 insertions(+), 45 deletions(-) create mode 100644 NEWS.md create mode 100644 man/bigrquerystorage-package.Rd delete mode 100644 man/bqs_ua.Rd diff --git a/DESCRIPTION b/DESCRIPTION index ab2fa45..176a490 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -15,7 +15,9 @@ Imports: DBI, bigrquery, assertthat, - Rcpp + Rcpp, + lifecycle, + bit64 Suggests: testthat, bit64, diff --git a/NAMESPACE b/NAMESPACE index 4d19618..5f4cf8c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -9,7 +9,12 @@ exportMethods(dbReadTable) import(DBI) import(bigrquery) import(methods) +importFrom(Rcpp,sourceCpp) importFrom(arrow,RecordBatchStreamReader) importFrom(arrow,Table) +importFrom(bit64,is.integer64) +importFrom(lifecycle,deprecate_warn) +importFrom(lifecycle,deprecated) importFrom(rlang,env_unlock) -useDynLib(bigrquerystorage) +importFrom(tibble,tibble) +useDynLib(bigrquerystorage, .registration = TRUE) diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..cfa2dbf --- /dev/null +++ b/NEWS.md @@ -0,0 +1,3 @@ +# bigrquerystorage 1.99.9000 + +* Initial CRAN submission. diff --git a/R/RcppExports.R b/R/RcppExports.R index 22190ed..7f0ecad 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -2,22 +2,22 @@ # Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 bqs_set_log_verbosity <- function(severity) { - invisible(.Call('_bigrquerystorage_bqs_set_log_verbosity', PACKAGE = 'bigrquerystorage', severity)) + invisible(.Call(`_bigrquerystorage_bqs_set_log_verbosity`, severity)) } bqs_init_logger <- function() { - invisible(.Call('_bigrquerystorage_bqs_init_logger', PACKAGE = 'bigrquerystorage')) + invisible(.Call(`_bigrquerystorage_bqs_init_logger`)) } grpc_version <- function() { - .Call('_bigrquerystorage_grpc_version', PACKAGE = 'bigrquerystorage') + .Call(`_bigrquerystorage_grpc_version`) } bqs_client <- function(client_info, service_configuration, refresh_token = "", access_token = "", root_certificate = "", target = "bigquerystorage.googleapis.com:443") { - .Call('_bigrquerystorage_bqs_client', PACKAGE = 'bigrquerystorage', client_info, service_configuration, refresh_token, access_token, root_certificate, target) + .Call(`_bigrquerystorage_bqs_client`, client_info, service_configuration, refresh_token, access_token, root_certificate, target) } bqs_ipc_stream <- function(client, project, dataset, table, parent, n, selected_fields, row_restriction = "", timestamp_seconds = 0L, timestamp_nanos = 0L, quiet = FALSE) { - .Call('_bigrquerystorage_bqs_ipc_stream', PACKAGE = 'bigrquerystorage', client, project, dataset, table, parent, n, selected_fields, row_restriction, timestamp_seconds, timestamp_nanos, quiet) + .Call(`_bigrquerystorage_bqs_ipc_stream`, client, project, dataset, table, parent, n, selected_fields, row_restriction, timestamp_seconds, timestamp_nanos, quiet) } diff --git a/R/bigrquerystorage.R b/R/bigrquerystorage.R index e751f70..cd269bd 100644 --- a/R/bigrquerystorage.R +++ b/R/bigrquerystorage.R @@ -1,6 +1,10 @@ +#' @aliases NULL +#' @aliases bigrquerystorage-package #' @import methods DBI -#' @useDynLib bigrquerystorage -NULL +#' @importFrom Rcpp sourceCpp +#' @importFrom bit64 is.integer64 +#' @useDynLib bigrquerystorage, .registration = TRUE +"_PACKAGE" .onLoad <- function(libname, pkgname) { # Setup grpc execution environment @@ -18,4 +22,4 @@ NULL dummy <- function() { Rcpp::compileAttributes -} \ No newline at end of file +} diff --git a/R/bqs_download.R b/R/bqs_download.R index aefc5eb..8fc8f59 100644 --- a/R/bqs_download.R +++ b/R/bqs_download.R @@ -16,20 +16,23 @@ #' which allows the full range of 64 bit integers. #' @export #' @importFrom arrow RecordBatchStreamReader Table +#' @importFrom lifecycle deprecated deprecate_warn +#' @importFrom tibble tibble bqs_table_download <- function( x, parent = getOption("bigquerystorage.project", ""), snapshot_time = NA, selected_fields = character(), row_restriction = "", - max_results = Inf, + n_max = Inf, quiet = NA, as_tibble = FALSE, - bigint = c("integer", "integer64", "numeric", "character")) { + bigint = c("integer", "integer64", "numeric", "character"), + max_results = lifecycle::deprecated()) { # Parameters validation bqs_table_name <- unlist(strsplit(unlist(x), "\\.|:")) - assertthat::assert_that(length(bqs_table_name) == 3) + assertthat::assert_that(length(bqs_table_name) >= 3) assertthat::assert_that(is.character(row_restriction)) assertthat::assert_that(is.character(selected_fields)) if (is.na(snapshot_time)) { @@ -39,12 +42,17 @@ bqs_table_download <- function( } timestamp_seconds <- as.integer(snapshot_time) timestamp_nanos <- as.integer(as.numeric(snapshot_time - timestamp_seconds)*1000000000) + if (lifecycle::is_present(max_results)) { + lifecycle::deprecate_warn("1.99.0", "bqs_table_download(max_results)", + "bqs_table_download(n_max)") + n_max <- max_results + } parent <- as.character(parent) if (nchar(parent) == 0) { parent <- bqs_table_name[1] } - if (max_results < 0 || max_results == Inf) { - max_results <- -1L + if (n_max < 0 || n_max == Inf) { + n_max <- -1L trim_to_n <- FALSE } else { trim_to_n <- TRUE @@ -62,7 +70,7 @@ bqs_table_download <- function( dataset = bqs_table_name[2], table = bqs_table_name[3], parent = parent, - n = max_results, + n = n_max, selected_fields = selected_fields, row_restriction = row_restriction, timestamp_seconds = timestamp_seconds, @@ -85,13 +93,18 @@ bqs_table_download <- function( } if (isTRUE(as_tibble)) { - tb <- asNamespace("bigrquery")$convert_bigint(as.data.frame(tb), bigint) + tb <- parse_postprocess( + tibble::tibble( + as.data.frame(tb) + ), + bigint + ) } - # Batches do not support a max_results so we get just enough results before + # Batches do not support a n_max so we get just enough results before # exiting the streaming loop. if (isTRUE(trim_to_n) && nrow(tb) > 0) { - tb <- tb[1:max_results, ] + tb <- tb[1:n_max, ] } return(tb) @@ -189,16 +202,22 @@ bqs_deauth <- function() { #' @export overload_bq_table_download <- function(parent) { utils::assignInNamespace("bq_table_download", function( - x, max_results = Inf, page_size = 10000, start_index = 0L, max_connections = 6L, - quiet = NA, bigint = c("integer", "integer64", "numeric", "character")) { + x, n_max = Inf, page_size = NULL, start_index = 0L, max_connections = 6L, + quiet = NA, bigint = c("integer", "integer64", "numeric", "character"), max_results = deprecated()) { + x <- bigrquery::as_bq_table(x) - assertthat::assert_that(is.numeric(max_results), length(max_results) == 1) + if (lifecycle::is_present(max_results)) { + lifecycle::deprecate_warn("1.4.0", "bq_table_download(max_results)", + "bq_table_download(n_max)") + n_max <- max_results + } + assertthat::assert_that(is.numeric(n_max), length(n_max) == 1) assertthat::assert_that(is.numeric(start_index), length(start_index) == 1) bigint <- match.arg(bigint) table_data <- bigrquerystorage::bqs_table_download( x = x, parent = parent, - max_results = max_results + start_index, + n_max = n_max + start_index, as_tibble = TRUE, quiet = quiet, bigint = bigint @@ -244,3 +263,32 @@ bqs_initiate <- function() { options("arrow.use_threads" = FALSE) } } + +# utils ------------------------------------------------------------------ +`%||%` <- function (x, y) if (is.null(x)) y else x + +parse_postprocess <- function(df, bigint) { + + if (bigint != "integer64") { + as_bigint <- switch( + bigint, + integer = as.integer, + numeric = as.numeric, + character = as.character + ) + df <- col_apply(df, bit64::is.integer64, as_bigint) + } + + df +} + +col_apply <- function(x, p, f) { + if (is.list(x)) { + x[] <- lapply(x, col_apply, p = p, f = f) + x + } else if (p(x)) { + f(x) + } else { + x + } +} diff --git a/R/client_info.R b/R/client_info.R index ec9944e..02a89d9 100644 --- a/R/client_info.R +++ b/R/client_info.R @@ -1,5 +1,5 @@ #' Loosely adapted from https://github.com/googleapis/python-api-core/blob/master/google/api_core/client_info.py -#' @noMd +#' @noRd bqs_ua <- function() { paste0( "bigrquerystorage", diff --git a/R/dbi.R b/R/dbi.R index 24771bd..e923fbf 100644 --- a/R/dbi.R +++ b/R/dbi.R @@ -20,7 +20,7 @@ setMethod( data <- bqs_table_download(res@bq_table, tryCatch(res@billing, error = function(e) {getOption("bigquerystorage.project","")}), - max_results = n + res@cursor$cur(), + n_max = n + res@cursor$cur(), as_tibble = TRUE, quiet = res@quiet, bigint = res@bigint, diff --git a/man/bigrquerystorage-package.Rd b/man/bigrquerystorage-package.Rd new file mode 100644 index 0000000..ddc7d12 --- /dev/null +++ b/man/bigrquerystorage-package.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/bigrquerystorage.R +\docType{package} +\name{bigrquerystorage-package} +\alias{bigrquerystorage-package} +\title{bigrquerystorage: R Client for BigQuery Storage API} +\description{ +Provides facilities to interact with BigQuery Storage API. +} +\seealso{ +Useful links: +\itemize{ + \item \url{https://github.com/meztez/bigrquerystorage} + \item Report bugs at \url{https://github.com/meztez/bigrquerystorage/issues} +} + +} +\author{ +\strong{Maintainer}: Bruno Tremblay \email{openr@neoxone.com} + +Other contributors: +\itemize{ + \item Google LLC [copyright holder, funder] +} + +} diff --git a/man/bqs_table_download.Rd b/man/bqs_table_download.Rd index 7425a6d..1b0e38d 100644 --- a/man/bqs_table_download.Rd +++ b/man/bqs_table_download.Rd @@ -10,10 +10,11 @@ bqs_table_download( snapshot_time = NA, selected_fields = character(), row_restriction = "", - max_results = Inf, + n_max = Inf, quiet = NA, as_tibble = FALSE, - bigint = c("integer", "integer64", "numeric", "character") + bigint = c("integer", "integer64", "numeric", "character"), + max_results = lifecycle::deprecated() ) } \arguments{ @@ -28,9 +29,6 @@ grpc method. You can set option `bigquerystorage.project`.} \item{row_restriction}{Restriction to apply to the table.} -\item{max_results}{Maximum number of results to retrieve. Use `Inf` or `-1L` -retrieve all rows.} - \item{quiet}{Should information be printed to console.} \item{as_tibble}{Should data be returned as tibble. Default is to return @@ -40,6 +38,9 @@ as arrow Table from raw IPC stream.} The default is `"integer"` which returns R's `integer` type but results in `NA` for values above/below +/- 2147483647. `"integer64"` returns a [bit64::integer64], which allows the full range of 64 bit integers.} + +\item{max_results}{Maximum number of results to retrieve. Use `Inf` or `-1L` +retrieve all rows.} } \description{ Download table from BigQuery using BigQuery Storage API diff --git a/man/bqs_ua.Rd b/man/bqs_ua.Rd deleted file mode 100644 index 58ed7d0..0000000 --- a/man/bqs_ua.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/client_info.R -\name{bqs_ua} -\alias{bqs_ua} -\title{Loosely adapted from https://github.com/googleapis/python-api-core/blob/master/google/api_core/client_info.py} -\usage{ -bqs_ua() -} -\description{ -Loosely adapted from https://github.com/googleapis/python-api-core/blob/master/google/api_core/client_info.py -} diff --git a/src/Makevars.in b/src/Makevars.in index 307275f..8e73b06 100644 --- a/src/Makevars.in +++ b/src/Makevars.in @@ -2,7 +2,7 @@ PKG_CPPFLAGS = @CPPF@ PKG_CXXFLAGS = @CXXF@ PKG_LIBS = @LIBS@ -CXX_STD = CXX17 +CXX_STD = CXX11 # Obtain the object files OBJECTS=@TARGETS@ diff --git a/tests/testthat/test-integration.R b/tests/testthat/test-integration.R index a6299e1..d061acd 100644 --- a/tests/testthat/test-integration.R +++ b/tests/testthat/test-integration.R @@ -3,8 +3,8 @@ test_that("BigQuery json and BigQuery return the same results", { # Compare with bigrquery method - dt <- bqs_table_download("bigquery-public-data.usa_names.usa_1910_current", bq_test_project(), max_results = 50000, as_tibble = TRUE, quiet = TRUE) - dt2 <- bq_table_download("bigquery-public-data.usa_names.usa_1910_current", max_results = 50000) + dt <- bqs_table_download("bigquery-public-data.usa_names.usa_1910_current", bq_test_project(), n_max = 50000, as_tibble = TRUE, quiet = TRUE) + dt2 <- bq_table_download("bigquery-public-data.usa_names.usa_1910_current", n_max = 50000, quiet = TRUE) expect_equal(dt, dt2) }) @@ -61,7 +61,7 @@ test_that("Optional BigQuery Storage API parameters work", { test_that("can read utf-8 strings", { sql <- "SELECT '\U0001f603' as x" - tb <- bq_project_query(bq_test_project(), sql) + tb <- bq_project_query(bq_test_project(), sql, quiet = TRUE) df <- bqs_table_download(tb, bq_test_project(), as_tibble = TRUE, quiet = TRUE) x <- df$x[[1]] @@ -82,10 +82,12 @@ test_that("can convert date time types", { tb <- bq_project_query(bq_test_project(), sql, quiet = TRUE) df <- bqs_table_download(tb, bq_test_project(), as_tibble = TRUE, quiet = TRUE) + df2 <- bq_table_download(tb, quiet = TRUE) base <- ISOdatetime(2000, 1, 2, 3, 4, 5.67, tz = "UTC") attr(df$datetime, "tzone") <- attr(base, "tzone") + expect_equal(df, df2, tolerance = 0.67) expect_equal(df$datetime, base) expect_equal(df$timestamp, base) expect_equal(df$date, as.Date(base))