Skip to content

Commit

Permalink
Merge pull request meztez#25 from gaborcsardi/fix/autobrew
Browse files Browse the repository at this point in the history
Add autobrew config
  • Loading branch information
meztez authored Dec 4, 2023
2 parents efb50b2 + a8fa4d3 commit 5196980
Show file tree
Hide file tree
Showing 6 changed files with 154 additions and 43 deletions.
2 changes: 2 additions & 0 deletions .Rbuildignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,5 @@
^README\.Rmd$
^\.github$
^LICENSE\.md$
^[.]deps$
^[.]deps-arm64$
19 changes: 12 additions & 7 deletions .github/workflows/R-CMD-check.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,12 @@ jobs:
fail-fast: false
matrix:
config:
- {os: macos-13, r: 'release'}
- { os: macos-11, r: 'release' }
- { os: macos-12, r: 'release' }
- { os: macos-13, r: 'release' }
- { os: macos-11, r: 'release', deps: true }
- { os: macos-12, r: 'release', deps: true }
- { os: macos-13, r: 'release', deps: true }

# - {os: windows-latest, r: 'release'}
# Use 3.6 to trigger usage of RTools35
Expand All @@ -31,11 +36,11 @@ jobs:
# - {os: windows-latest, r: '4.1'}

# - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'}
- {os: ubuntu-latest, r: 'release'}
- {os: ubuntu-latest, r: 'oldrel-1'}
- {os: ubuntu-latest, r: 'oldrel-2'}
- {os: ubuntu-latest, r: 'oldrel-3'}
- {os: ubuntu-latest, r: 'oldrel-4'}
- { os: ubuntu-latest, r: 'release' }
- { os: ubuntu-latest, r: 'oldrel-1' }
- { os: ubuntu-latest, r: 'oldrel-2' }
- { os: ubuntu-latest, r: 'oldrel-3' }
- { os: ubuntu-latest, r: 'oldrel-4' }

env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
Expand Down Expand Up @@ -63,7 +68,7 @@ jobs:
pkg-config
- name: "System packages on macOS"
if: runner.os == 'macOS'
if: runner.os == 'macOS' && matrix.config.deps
run: |
brew install grpc pkg-config
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,4 @@ src/google/
.httr-oauth
/src/Makevars
/.deps
/.deps-arm64
1 change: 1 addition & 0 deletions src/bqs.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include <grpcpp/grpcpp.h>
#include "google/cloud/bigquery/storage/v1/stream.pb.h"
#include "google/cloud/bigquery/storage/v1/storage.pb.h"
# pragma GCC diagnostic ignored "-Winconsistent-missing-override"
#include "google/cloud/bigquery/storage/v1/storage.grpc.pb.h"
#include <Rcpp.h>
#include "RProgress.h"
Expand Down
58 changes: 22 additions & 36 deletions tools/config/configure.R
Original file line number Diff line number Diff line change
Expand Up @@ -2,56 +2,44 @@
# Use 'define()' to define configuration variables.
# Use 'configure_file()' to substitute configuration values.

source("tools/config/functions.R")

# Check OS ----------------------------------------------------------------
win <- .Platform$OS.type == "windows"
mac <- Sys.info()[["sysname"]] == "Darwin"

sys <- function(cmd, intern = TRUE, ...) {
suppressWarnings(system(cmd, intern = intern, ...))
as_string(suppressWarnings(system(cmd, intern = intern, ...)))
}

# System requirements -----------------------------------------------------

done <- FALSE
config_from <- NULL

# flags specified explicitly
cflags <- Sys.getenv("BQS_CFLAGS")
ldflags <- Sys.getenv("BQS_LDFLAGS")
done <- nzchar(cflags) || nzchar(ldflags)
if (done) config_from <- "explicit"

# working pkg-config finds the required libs
if (!done && Sys.getenv("AUTOBREW_FORCE") == "" &&
nzchar(Sys.which("pkg-config"))) {
cflags <- sys("pkg-config --cflags --silence-errors grpc++ protobuf")
ldflags <- sys("pkg-config --cflags --libs grpc++ protobuf")
ldflags <- sys("pkg-config --libs --silence-errors grpc++ protobuf")
done <- nzchar(cflags) || nzchar(ldflags)
if (done) config_from <- "pkg-config"
}

# autobrew on mac
if (!done && mac && Sys.getenv("DISABLE_AUTOBREW") == "") {
deps <- c(
"grpc-static-1.59.3",
"re2-static-20231101",
"protobuf-static-25.1",
"openssl-static-3.1.1",
"jsoncpp-static-1.9.5",
"c-ares-static-1.22.1",
"abseil-static-20230802.1"
)
plfm <- if (R.Version()$arch == "aarch64") "arm64_big_sur" else "big_sur"

repo <- "https://github.com/gaborcsardi/homebrew-cran"
urls <- sprintf(
"%s/releases/download/%s/%s.%s.bottle.tar.gz",
repo, deps, deps, plfm
)
for (url in urls) {
tgt <- file.path(".deps", basename(url))
dir.create(dirname(tgt), showWarnings = FALSE, recursive = TRUE)
if (file.exists(tgt)) next
download.file(url, tgt, quiet = TRUE)
untar(tgt, exdir = ".deps")
}
download_autobrew()
flags <- configure_autobrew()
cflags <- as_string(flags$cflags)
ldflags <- as_string(flags$ldflags)
done <- nzchar(cflags) || nzchar(ldflags)
if (done) config_from <- "autobrew"
}

# TODO: download static libs on windows
Expand All @@ -69,15 +57,10 @@ detect_binary <- function(binary) {
message(sprintf("*** searching for %s ...", binary), appendLF = FALSE)
path <- Sys.which(binary)
if (path == "") {
if (binary == "pkg-config") {
install_with_pacman("pkgconf", RTOOLS43_ROOT, WIN)
path <- Sys.which(binary)
} else {
message(" Failed")
stop("Could not find ", binary)
}
message(" Failed")
stop("Could not find ", binary)
} else {
message(" OK")
message(" OK: ", path)
}
path
}
Expand Down Expand Up @@ -126,7 +109,10 @@ compile_order <- function(pbpath, bpath) {
protos <- compile_order(services, base_proto_path)

# protos include path (to locate google/protobuf/*.proto)
ipath <- base_proto_path
ipath <- c(
base_proto_path,
if (config_from == "autobrew") autobrew_proto_include_path
)

# compile proto files to generate basic grpc client
message("*** compiling proto files ...")
Expand Down Expand Up @@ -196,7 +182,7 @@ for (src_ in gle_cc) {
pkg_sources <- sort(dir("./src", ".cpp$|.c$"), decreasing = TRUE)

# compiler flags
cxxflags <- "-I."
cxxflags <- if (mac) "-std=gnu++17" else ""

fix_flags <- function(x) {
x <- gsub("-Wno-float-conversion ", "", x, fixed = TRUE)
Expand All @@ -214,7 +200,7 @@ fix_flags <- function(x) {
}

# define variable for template
define(CPPF = fix_flags(paste(cflags, "-DSTRING_R_HEADERS")))
define(CPPF = fix_flags(paste(cflags, "-DSTRING_R_HEADERS", "-I.")))
define(CXXF = cxxflags)
define(LIBS = fix_flags(ldflags))
define(TARGETS = paste(c(
Expand Down
116 changes: 116 additions & 0 deletions tools/config/functions.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
as_string <- function(x) {
paste(x, collapse = "")
}

# autobrew
autobrew_path <- if (R.Version()$arch == "aarch64") ".deps-arm64" else ".deps"
autobrew_proto_include_path <- file.path(
autobrew_path,
"protobuf-static",
"25.1",
"include"
)

# autobrew on mac, download
download_autobrew <- function() {
path <- autobrew_path
deps <- c(
"pkg-config-0.29.2_3",
"grpc-static-1.59.3",
"re2-static-20231101",
"protobuf-static-25.1",
"openssl-static-3.1.1",
"jsoncpp-static-1.9.5",
"c-ares-static-1.22.1",
"abseil-static-20230802.1"
)
plfm <- if (R.Version()$arch == "aarch64") "arm64_big_sur" else "big_sur"

repo <- "https://github.com/gaborcsardi/homebrew-cran"
urls <- sprintf(
"%s/releases/download/%s/%s.%s.bottle.tar.gz",
repo, sub("_[0-9]+$", "", deps), deps, plfm
)
for (url in urls) {
tgt <- file.path(path, basename(url))
dir.create(dirname(tgt), showWarnings = FALSE, recursive = TRUE)
if (file.exists(tgt)) next
download.file(url, tgt, quiet = TRUE)
untar(tgt, exdir = path)
}
}

# autobrew on mac, configure
configure_autobrew <- function() {
path <- autobrew_path
# e.g. .deps/protobuf-static/25.1/
pkg_dirs <- file.path(
path,
Filter(function(x) {
file.info(file.path(path, x))$isdir
}, setdiff(dir(path), c("pkgconfig", "bin")))
)

# e.g. .deps/protobuf-static/25.1/INSTALL_RECEIPT.
# ocntains the files that have @@HOMEBREW_*@@ substitutions
rcpt <- file.path(
pkg_dirs,
vapply(pkg_dirs, dir, character(1)),
"INSTALL_RECEIPT.json"
)
# we only need to replace @@HOMEBREW_CELLAR@@
cellar <- file.path(getwd(), path)
for (js in rcpt) {
lns <- readLines(js, warn = FALSE)
from <- grep("changed_files", lns)[1] + 1L
if (is.na(from)) next
lns <- lns[from:length(lns)]
to <- grep("^\\s*],\\s*$", lns)[1] - 1L
if (is.na(to)) next
lns <- sub("\",?$", "", sub("^\"", "", trimws(lns[1:to])))
cfs <- file.path(dirname(js), lns)
for (cf in cfs) {
cflns <- readLines(cf, warn = FALSE)
cflns <- gsub("@@HOMEBREW_CELLAR@@", cellar, cflns, fixed = TRUE)
Sys.chmod(cf, "0664")
writeLines(cflns, cf)
}
}

# copy pkg-config .pc files to a single directory, then only
# one directory is on PKG_CONFIG_PATH, and easier to make sure that
# `normalizePath()` works.
pcdir <- file.path(path, "pkgconfig")
unlink(pcdir, recursive = TRUE, force = TRUE)
dir.create(pcdir, recursive = TRUE)
pcdirs <- file.path(dirname(rcpt), "lib", "pkgconfig")
for (pc1 in pcdirs) {
file.copy(file.path(pc1, dir(pc1)), pcdir)
}
Sys.setenv(PKG_CONFIG_PATH = normalizePath(pcdir))

# copy all binaries to a single directory and put that at the
# beginning pf PATH, so our version of `protoc` is found first
bindir <- file.path(path, "bin")
unlink(bindir, recursive = TRUE, force = TRUE)
dir.create(bindir, recursive = TRUE)
bindirs <- file.path(dirname(rcpt), "bin")
for (b1 in bindirs) {
file.copy(file.path(b1, dir(b1)), bindir)
}
paths <- strsplit(
Sys.getenv("PATH"),
.Platform$path.sep,
fixed = TRUE
)[[1]]
paths <- c(normalizePath(bindir), paths)
Sys.setenv(PATH = paste(paths, collapse = .Platform$path.sep))

list(
cflags = sys(
"pkg-config --cflags --silence-errors grpc++ protobuf"),
ldflags = sys(
"pkg-config --libs --static --silence-errors grpc++ protobuf"
)
)
}

0 comments on commit 5196980

Please sign in to comment.