From 277076bb892c9482cd9ee40a0ecbd4f631cb1b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Mon, 4 Dec 2023 09:24:45 +0100 Subject: [PATCH 1/5] Add autobrew config Download system requirements on macOS, if they are not available locally. --- .Rbuildignore | 1 + tools/config/configure.R | 58 ++++++++------------ tools/config/functions.R | 116 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 36 deletions(-) create mode 100644 tools/config/functions.R diff --git a/.Rbuildignore b/.Rbuildignore index de806fe..c4ebe9e 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -9,3 +9,4 @@ ^README\.Rmd$ ^\.github$ ^LICENSE\.md$ +^[.]deps$ diff --git a/tools/config/configure.R b/tools/config/configure.R index 22e5b20..cef801f 100644 --- a/tools/config/configure.R +++ b/tools/config/configure.R @@ -2,56 +2,44 @@ # Use 'define()' to define configuration variables. # Use 'configure_file()' to substitute configuration values. +source("tools/config/functions.R") + # Check OS ---------------------------------------------------------------- win <- .Platform$OS.type == "windows" mac <- Sys.info()[["sysname"]] == "Darwin" sys <- function(cmd, intern = TRUE, ...) { - suppressWarnings(system(cmd, intern = intern, ...)) + as_string(suppressWarnings(system(cmd, intern = intern, ...))) } # System requirements ----------------------------------------------------- done <- FALSE +config_from <- NULL # flags specified explicitly cflags <- Sys.getenv("BQS_CFLAGS") ldflags <- Sys.getenv("BQS_LDFLAGS") done <- nzchar(cflags) || nzchar(ldflags) +if (done) config_from <- "explicit" # working pkg-config finds the required libs if (!done && Sys.getenv("AUTOBREW_FORCE") == "" && nzchar(Sys.which("pkg-config"))) { cflags <- sys("pkg-config --cflags --silence-errors grpc++ protobuf") - ldflags <- sys("pkg-config --cflags --libs grpc++ protobuf") + ldflags <- sys("pkg-config --libs --silence-errors grpc++ protobuf") done <- nzchar(cflags) || nzchar(ldflags) + if (done) config_from <- "pkg-config" } # autobrew on mac if (!done && mac && Sys.getenv("DISABLE_AUTOBREW") == "") { - deps <- c( - "grpc-static-1.59.3", - "re2-static-20231101", - "protobuf-static-25.1", - "openssl-static-3.1.1", - "jsoncpp-static-1.9.5", - "c-ares-static-1.22.1", - "abseil-static-20230802.1" - ) - plfm <- if (R.Version()$arch == "aarch64") "arm64_big_sur" else "big_sur" - - repo <- "https://github.com/gaborcsardi/homebrew-cran" - urls <- sprintf( - "%s/releases/download/%s/%s.%s.bottle.tar.gz", - repo, deps, deps, plfm - ) - for (url in urls) { - tgt <- file.path(".deps", basename(url)) - dir.create(dirname(tgt), showWarnings = FALSE, recursive = TRUE) - if (file.exists(tgt)) next - download.file(url, tgt, quiet = TRUE) - untar(tgt, exdir = ".deps") - } + download_autobrew() + flags <- configure_autobrew() + cflags <- as_string(flags$cflags) + ldflags <- as_string(flags$ldflags) + done <- nzchar(cflags) || nzchar(ldflags) + if (done) config_from <- "autobrew" } # TODO: download static libs on windows @@ -69,15 +57,10 @@ detect_binary <- function(binary) { message(sprintf("*** searching for %s ...", binary), appendLF = FALSE) path <- Sys.which(binary) if (path == "") { - if (binary == "pkg-config") { - install_with_pacman("pkgconf", RTOOLS43_ROOT, WIN) - path <- Sys.which(binary) - } else { - message(" Failed") - stop("Could not find ", binary) - } + message(" Failed") + stop("Could not find ", binary) } else { - message(" OK") + message(" OK: ", path) } path } @@ -126,7 +109,10 @@ compile_order <- function(pbpath, bpath) { protos <- compile_order(services, base_proto_path) # protos include path (to locate google/protobuf/*.proto) -ipath <- base_proto_path +ipath <- c( + base_proto_path, + if (config_from == "autobrew") autobrew_proto_include_path +) # compile proto files to generate basic grpc client message("*** compiling proto files ...") @@ -196,7 +182,7 @@ for (src_ in gle_cc) { pkg_sources <- sort(dir("./src", ".cpp$|.c$"), decreasing = TRUE) # compiler flags -cxxflags <- "-I." +cxxflags <- "" fix_flags <- function(x) { x <- gsub("-Wno-float-conversion ", "", x, fixed = TRUE) @@ -214,7 +200,7 @@ fix_flags <- function(x) { } # define variable for template -define(CPPF = fix_flags(paste(cflags, "-DSTRING_R_HEADERS"))) +define(CPPF = fix_flags(paste(cflags, "-DSTRING_R_HEADERS", "-I."))) define(CXXF = cxxflags) define(LIBS = fix_flags(ldflags)) define(TARGETS = paste(c( diff --git a/tools/config/functions.R b/tools/config/functions.R new file mode 100644 index 0000000..0e13f66 --- /dev/null +++ b/tools/config/functions.R @@ -0,0 +1,116 @@ +as_string <- function(x) { + paste(x, collapse = "") +} + +# autobrew +autobrew_path <- ".deps" +autobrew_proto_include_path <- file.path( + autobrew_path, + "protobuf-static", + "25.1", + "include" +) + +# autobrew on mac, download +download_autobrew <- function() { + path <- autobrew_path + deps <- c( + "pkg-config-0.29.2_3", + "grpc-static-1.59.3", + "re2-static-20231101", + "protobuf-static-25.1", + "openssl-static-3.1.1", + "jsoncpp-static-1.9.5", + "c-ares-static-1.22.1", + "abseil-static-20230802.1" + ) + plfm <- if (R.Version()$arch == "aarch64") "arm64_big_sur" else "big_sur" + + repo <- "https://github.com/gaborcsardi/homebrew-cran" + urls <- sprintf( + "%s/releases/download/%s/%s.%s.bottle.tar.gz", + repo, sub("_[0-9]+$", "", deps), deps, plfm + ) + for (url in urls) { + tgt <- file.path(path, basename(url)) + dir.create(dirname(tgt), showWarnings = FALSE, recursive = TRUE) + if (file.exists(tgt)) next + download.file(url, tgt, quiet = TRUE) + untar(tgt, exdir = path) + } +} + +# autobrew on mac, configure +configure_autobrew <- function() { + path <- autobrew_path + # e.g. .deps/protobuf-static/25.1/ + pkg_dirs <- file.path( + path, + Filter(function(x) { + file.info(file.path(path, x))$isdir + }, setdiff(dir(path), c("pkgconfig", "bin"))) + ) + + # e.g. .deps/protobuf-static/25.1/INSTALL_RECEIPT. + # ocntains the files that have @@HOMEBREW_*@@ substitutions + rcpt <- file.path( + pkg_dirs, + vapply(pkg_dirs, dir, character(1)), + "INSTALL_RECEIPT.json" + ) + # we only need to replace @@HOMEBREW_CELLAR@@ + cellar <- file.path(getwd(), path) + for (js in rcpt) { + lns <- readLines(js, warn = FALSE) + from <- grep("changed_files", lns)[1] + 1L + if (is.na(from)) next + lns <- lns[from:length(lns)] + to <- grep("^\\s*],\\s*$", lns)[1] - 1L + if (is.na(to)) next + lns <- sub("\",?$", "", sub("^\"", "", trimws(lns[1:to]))) + cfs <- file.path(dirname(js), lns) + for (cf in cfs) { + cflns <- readLines(cf, warn = FALSE) + cflns <- gsub("@@HOMEBREW_CELLAR@@", cellar, cflns, fixed = TRUE) + Sys.chmod(cf, "0664") + writeLines(cflns, cf) + } + } + + # copy pkg-config .pc files to a single directory, then only + # one directory is on PKG_CONFIG_PATH, and easier to make sure that + # `normalizePath()` works. + pcdir <- file.path(path, "pkgconfig") + unlink(pcdir, recursive = TRUE, force = TRUE) + dir.create(pcdir, recursive = TRUE) + pcdirs <- file.path(dirname(rcpt), "lib", "pkgconfig") + for (pc1 in pcdirs) { + file.copy(file.path(pc1, dir(pc1)), pcdir) + } + Sys.setenv(PKG_CONFIG_PATH = normalizePath(pcdir)) + + # copy all binaries to a single directory and put that at the + # beginning pf PATH, so our version of `protoc` is found first + bindir <- file.path(path, "bin") + unlink(bindir, recursive = TRUE, force = TRUE) + dir.create(bindir, recursive = TRUE) + bindirs <- file.path(dirname(rcpt), "bin") + for (b1 in bindirs) { + file.copy(file.path(b1, dir(b1)), bindir) + } + paths <- strsplit( + Sys.getenv("PATH"), + .Platform$path.sep, + fixed = TRUE + )[[1]] + paths <- c(normalizePath(bindir), paths) + Sys.setenv(PATH = paste(paths, collapse = .Platform$path.sep)) + + list( + cflags = sys( + "pkg-config --cflags --silence-errors grpc++ protobuf"), + ldflags = sys( + "pkg-config --libs --static --silence-errors grpc++ protobuf" + ) + ) +} From e9107aa2af0c9d2841b15c4e1d5a686216798759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Mon, 4 Dec 2023 09:25:36 +0100 Subject: [PATCH 2/5] Fix R CMD check warning for autobrew Warning coming from the system headers. --- src/bqs.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bqs.cpp b/src/bqs.cpp index 2ee2df9..71e5ba2 100644 --- a/src/bqs.cpp +++ b/src/bqs.cpp @@ -6,6 +6,7 @@ #include #include "google/cloud/bigquery/storage/v1/stream.pb.h" #include "google/cloud/bigquery/storage/v1/storage.pb.h" +# pragma GCC diagnostic ignored "-Winconsistent-missing-override" #include "google/cloud/bigquery/storage/v1/storage.grpc.pb.h" #include #include "RProgress.h" From e3ab944da1145396eac15fadd3779ab525ce7fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Mon, 4 Dec 2023 09:25:58 +0100 Subject: [PATCH 3/5] Run autobrew on CI --- .github/workflows/R-CMD-check.yaml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 09bbded..f81c89a 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -22,7 +22,12 @@ jobs: fail-fast: false matrix: config: - - {os: macos-13, r: 'release'} + - { os: macos-11, r: 'release' } + - { os: macos-12, r: 'release' } + - { os: macos-13, r: 'release' } + - { os: macos-11, r: 'release', deps: true } + - { os: macos-12, r: 'release', deps: true } + - { os: macos-13, r: 'release', deps: true } # - {os: windows-latest, r: 'release'} # Use 3.6 to trigger usage of RTools35 @@ -31,11 +36,11 @@ jobs: # - {os: windows-latest, r: '4.1'} # - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} - - {os: ubuntu-latest, r: 'release'} - - {os: ubuntu-latest, r: 'oldrel-1'} - - {os: ubuntu-latest, r: 'oldrel-2'} - - {os: ubuntu-latest, r: 'oldrel-3'} - - {os: ubuntu-latest, r: 'oldrel-4'} + - { os: ubuntu-latest, r: 'release' } + - { os: ubuntu-latest, r: 'oldrel-1' } + - { os: ubuntu-latest, r: 'oldrel-2' } + - { os: ubuntu-latest, r: 'oldrel-3' } + - { os: ubuntu-latest, r: 'oldrel-4' } env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} @@ -63,7 +68,7 @@ jobs: pkg-config - name: "System packages on macOS" - if: runner.os == 'macOS' + if: runner.os == 'macOS' && matrix.config.deps run: | brew install grpc pkg-config From ca48314dec3313fcf66cc6a205b3f21a0269345e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Mon, 4 Dec 2023 11:11:11 +0100 Subject: [PATCH 4/5] Autobrew: use different directory for arm64 sources To make it easier to run x86_64 R on arm64 machines. [ci skip] --- .Rbuildignore | 1 + .gitignore | 1 + tools/config/functions.R | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.Rbuildignore b/.Rbuildignore index c4ebe9e..6b2bddb 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -10,3 +10,4 @@ ^\.github$ ^LICENSE\.md$ ^[.]deps$ +^[.]deps-arm64$ diff --git a/.gitignore b/.gitignore index a7d352b..7d72e5d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ src/google/ .httr-oauth /src/Makevars /.deps +/.deps-arm64 diff --git a/tools/config/functions.R b/tools/config/functions.R index 0e13f66..09819ec 100644 --- a/tools/config/functions.R +++ b/tools/config/functions.R @@ -3,7 +3,7 @@ as_string <- function(x) { } # autobrew -autobrew_path <- ".deps" +autobrew_path <- if (R.Version()$arch == "aarch64") ".deps-arm64" else ".deps" autobrew_proto_include_path <- file.path( autobrew_path, "protobuf-static", From a8fa4d318f46a877f31bf6d64339e7e81cd01561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Mon, 4 Dec 2023 11:22:50 +0100 Subject: [PATCH 5/5] Require C++17 on macOS To be able to install the package on older R versions, e.g. R 3.6 to R 4.2, which default to C++14 or C++11. This is surely ok for macOS systems since Big Sur. --- tools/config/configure.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/config/configure.R b/tools/config/configure.R index cef801f..a76102b 100644 --- a/tools/config/configure.R +++ b/tools/config/configure.R @@ -182,7 +182,7 @@ for (src_ in gle_cc) { pkg_sources <- sort(dir("./src", ".cpp$|.c$"), decreasing = TRUE) # compiler flags -cxxflags <- "" +cxxflags <- if (mac) "-std=gnu++17" else "" fix_flags <- function(x) { x <- gsub("-Wno-float-conversion ", "", x, fixed = TRUE)