Refine build_arrow_static.sh and add wiring in r/configure to consume…

… the static lib

python/manylinux201x/build_arrow_static.sh

@@ -26,65 +26,49 @@
 #   or pull:
 #     $ docker-compose pull centos-python-manylinux2010
 #   and then run:
-#     $ docker-compose run -e PYTHON_VERSION=3.7 centos-python-manylinux2010
+#     $ docker-compose run r-manylinux
 # Can use either manylinux2010 or manylinux2014
 
-source /multibuild/manylinux_utils.sh
-
 # Quit on failure
 set -e
 
 # Print commands for debugging
-# set -x
+set -x
 
 cd /arrow/python
 
-export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/arrow-dist/lib/pkgconfig
-
-# Ensure the target directory exists
-mkdir -p /io/dist
-
-ARROW_BUILD_DIR=/io/dist
+ARROW_BUILD_DIR="$(pwd)/manylinux201x/dist"
 mkdir -p "${ARROW_BUILD_DIR}"
 pushd "${ARROW_BUILD_DIR}"
-CC=gcc CXX=g++ cmake -DCMAKE_BUILD_TYPE=Release \
+cmake -DCMAKE_BUILD_TYPE=Release \
     -DARROW_DEPENDENCY_SOURCE=BUNDLED \
-    -DZLIB_ROOT=/usr/local \
-    -DCMAKE_INSTALL_PREFIX=/io/dist \
+    -DCMAKE_INSTALL_PREFIX=${ARROW_BUILD_DIR} \
     -DCMAKE_INSTALL_LIBDIR=lib \
     -DARROW_BUILD_TESTS=OFF \
     -DARROW_BUILD_SHARED=OFF \
     -DARROW_BUILD_STATIC=ON \
     -DARROW_BOOST_USE_SHARED=OFF \
-    -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-8/root/usr/include/c++/8/;-isystem;/opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/" \
     -DARROW_JEMALLOC=ON \
-    -DARROW_RPATH_ORIGIN=ON \
-    -DARROW_PYTHON=OFF \
     -DARROW_COMPUTE=ON \
     -DARROW_CSV=ON \
     -DARROW_FILESYSTEM=ON \
     -DARROW_JSON=ON \
     -DARROW_PARQUET=ON \
     -DARROW_DATASET=ON \
-    -DARROW_PLASMA=OFF \
-    -DARROW_TENSORFLOW=OFF \
     -DARROW_ORC=OFF \
     -DARROW_WITH_BZ2=ON \
     -DARROW_WITH_ZLIB=ON \
     -DARROW_WITH_ZSTD=ON \
     -DARROW_WITH_LZ4=ON \
     -DARROW_WITH_SNAPPY=ON \
     -DARROW_WITH_BROTLI=ON \
-    -DARROW_FLIGHT=OFF \
-    -DARROW_GANDIVA=OFF \
-    -DARROW_GANDIVA_JAVA=OFF \
-    -DBoost_NAMESPACE=arrow_boost \
-    -DBOOST_ROOT=/arrow_boost_dist \
     -DOPENSSL_USE_STATIC_LIBS=ON \
     -GNinja /arrow/cpp
 ninja -v install
 
 # Copy the bundled static libs from the build to the install dir
 find . -regex .*/lib/.*\\.a\$ | xargs -I{} cp {} ./lib
 
+zip -r libarrow.zip lib/*.a include/
+
 popd

r/.Rbuildignore

@@ -8,6 +8,7 @@ lint.sh
 Dockerfile
 .*\.tar\.gz
 ^windows
+^libarrow
 clang_format.sh
 ^cran-comments\.md$
 ^arrow_.*.tar.gz$

r/.gitignore

@@ -12,5 +12,6 @@ inst/doc
 src/Makevars
 src/Makevars.win
 windows/
+libarrow/
 arrow_*.tar.gz
 arrow_*.tgz

r/configure

@@ -31,12 +31,7 @@ PKG_DEB_NAME="(unsuppored)"
 PKG_RPM_NAME="(unsuppored)"
 PKG_BREW_NAME="apache-arrow"
 PKG_TEST_HEADER="<arrow/api.h>"
-PKG_LIBS="-larrow -lparquet -larrow_dataset"
-
-# Temporarily put here
-if [ "$LIBARROW_BINARY" ]; then
-  PKG_LIBS="$PKG_LIBS -lthrift -lsnappy -lz -lzstd -llz4 -lbrotlidec-static -lbrotlienc-static -lbrotlicommon-static -lboost_filesystem -lboost_regex -lboost_system -ljemalloc_pic"
-fi
+PKG_LIBS="-lparquet -larrow_dataset -larrow"
 
 # generate code
 if [ "$ARROW_R_DEV" = "TRUE" ]; then
@@ -65,7 +60,7 @@ if [ "$INCLUDE_DIR" ] || [ "$LIB_DIR" ]; then
 else
   # Use pkg-config if available
   pkg-config --version >/dev/null 2>&1
-  if [ "$FORCE_AUTOBREW" != "TRUE" ] && [ $? -eq 0 ]; then
+  if [ "$LOCAL_LIBARROW" = "" ] && [ "$FORCE_AUTOBREW" != "TRUE" ] && [ $? -eq 0 ]; then
     PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}`
     PKGCONFIG_LIBS=`pkg-config --libs --silence-errors ${PKG_CONFIG_NAME}`
   fi
@@ -96,6 +91,13 @@ else
       fi
       PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include"
       PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS"
+    else
+      VERSION=$(grep ^Version DESCRIPTION | sed s/Version:\ //)
+      ${R_HOME}/bin/Rscript tools/linuxlibs.R $VERSION $LOCAL_LIBARROW
+      PKG_CFLAGS="-I$(pwd)/libarrow/arrow-${VERSION}/include $PKG_CFLAGS"
+      PKG_LIBS="-L$(pwd)/libarrow/arrow-${VERSION}/lib $PKG_LIBS"
+      # Also enumerate the static libs included in there
+      PKG_LIBS="$PKG_LIBS -lthrift -lsnappy -lz -lzstd -llz4 -lbrotlidec-static -lbrotlienc-static -lbrotlicommon-static -lboost_filesystem -lboost_regex -lboost_system -ljemalloc_pic"
     fi
   fi
 fi
@@ -109,6 +111,8 @@ CPPFLAGS=`"${R_HOME}"/bin/R CMD config CPPFLAGS`
 # If libarrow uses the old GLIBCXX ABI, so we have to use it too
 if [ "$ARROW_USE_OLD_CXXABI" ]; then
   PKG_CFLAGS="$PKG_CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
+# else
+#   PKG_CFLAGS="$PKG_CFLAGS -D_GLIBCXX_USE_CXX11_ABI=1"
 fi
 
 # Set any user-defined CXXFLAGS

r/tools/linuxlibs.R

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+args <- commandArgs(TRUE)
+VERSION <- args[1]
+dst_dir <- paste0("libarrow/arrow-", VERSION)
+if (!file.exists(paste0(dst_dir, "/include/arrow/api.h"))) {
+  if (length(args) > 1) {
+    # Arg 2 would be the path/to/lib.zip
+    localfile <- args[2]
+    cat(sprintf("*** Using LOCAL_LIBARROW %s\n", localfile))
+    if(!file.exists(localfile)){
+      cat(sprintf("*** %s does not exist; build will fail\n", localfile))
+    }
+    file.copy(localfile, "lib.zip")
+  } else {
+    # Download static arrow
+    try(download.file("https://dl.bintray.com/nealrichardson/pyarrow-dev/libarrow.zip", "lib.zip", quiet = TRUE), silent = TRUE)
+  }
+  dir.create(dst_dir, showWarnings = FALSE, recursive = TRUE)
+  unzip("lib.zip", exdir = dst_dir)
+  unlink("lib.zip")
+}