From 79f4b6bf4f1396ba35191b575f2ffd055664c17c Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 4 Sep 2022 10:48:31 +0200 Subject: [PATCH 01/22] Another try at building Arrow. --- A/Arrow/build_tarballs.jl | 56 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 A/Arrow/build_tarballs.jl diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl new file mode 100644 index 00000000000..7e1ef35f5de --- /dev/null +++ b/A/Arrow/build_tarballs.jl @@ -0,0 +1,56 @@ +# Note that this script can accept some limited command-line arguments, run +# `julia build_tarballs.jl --help` to see a usage message. +using BinaryBuilder, Pkg + +name = "Arrow" +version = v"9.0.0" + +# Collection of sources required to complete build +sources = [ + ArchiveSource("https://github.com/apache/arrow/archive/refs/tags/apache-arrow-9.0.0.zip", "fb2469e9bfeb3e9a45f1e3086f536a329d59a0efd3c905436d3b6e8dd030be41") +] + +# Bash recipe for building across all platforms +script = raw""" + +cd $WORKSPACE/srcdir/arrow-apache-arrow-*/cpp + +CMAKE_FLAGS=(-DCMAKE_INSTALL_PREFIX=$prefix +-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} +-DCMAKE_BUILD_TYPE=Release +-DARROW_BUILD_UTILITIES=ON +-DARROW_BUILD_STATIC=OFF +-DARROW_DATASET=ON +-DARROW_WITH_BZ2=ON +-DARROW_WITH_LZ4=ON +-DARROW_WITH_ZSTD=ON +-DARROW_WITH_ZLIB=ON +-DARROW_WITH_SNAPPY=ON +-DARROW_PARQUET=ON) + +cmake . ${CMAKE_FLAGS[@]} + +make -j${nproc} +make install +""" + +# These are the platforms we will build for by default, unless further +# platforms are passed in on the command line +platforms = expand_cxxstring_abis(supported_platforms()) + +# The products that we will ensure are always built +products = [ + LibraryProduct("libparquet", :libparquet), + LibraryProduct("libarrow", :libarrow) +] + +# Dependencies that must be installed before this package can be built +dependencies = Dependency[ + Dependency("boost_jll") + Dependency("Zlib_jll") + Dependency("Zstd_jll") + Dependency("snappy_jll") +] + +# Build the tarballs, and possibly a `build.jl` as well. +build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6") From 1f072cee67564957c12f5518dccb3c7217c22985 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 4 Sep 2022 11:03:03 +0200 Subject: [PATCH 02/22] Add compression deps. --- A/Arrow/build_tarballs.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 7e1ef35f5de..5372702d2d4 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -48,7 +48,9 @@ products = [ dependencies = Dependency[ Dependency("boost_jll") Dependency("Zlib_jll") + Dependency("Bzip2_jll") Dependency("Zstd_jll") + Dependency("Lz4_jll") Dependency("snappy_jll") ] From 6739697abbd322fdeb20b4703cdd270b3b6f71ce Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 4 Sep 2022 11:15:33 +0200 Subject: [PATCH 03/22] Add jemalloc dep. --- A/Arrow/build_tarballs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 5372702d2d4..55f973602e6 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -51,7 +51,7 @@ dependencies = Dependency[ Dependency("Bzip2_jll") Dependency("Zstd_jll") Dependency("Lz4_jll") - Dependency("snappy_jll") + Dependency("jemalloc_jll") ] # Build the tarballs, and possibly a `build.jl` as well. From 0830ba4218e2cae922f67633c6ce96b6a8c67844 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 4 Sep 2022 11:31:26 +0200 Subject: [PATCH 04/22] Set compat for deps and try to use jemalloc. --- A/Arrow/build_tarballs.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 55f973602e6..1ab1883d162 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -26,6 +26,7 @@ CMAKE_FLAGS=(-DCMAKE_INSTALL_PREFIX=$prefix -DARROW_WITH_ZSTD=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_SNAPPY=ON +-DARROW_JEMALLOC_USE_SHARED=ON -DARROW_PARQUET=ON) cmake . ${CMAKE_FLAGS[@]} @@ -46,9 +47,9 @@ products = [ # Dependencies that must be installed before this package can be built dependencies = Dependency[ - Dependency("boost_jll") + Dependency("boost_jll", compat="=1.76.0") Dependency("Zlib_jll") - Dependency("Bzip2_jll") + Dependency("Bzip2_jll", compat="1.0.7") Dependency("Zstd_jll") Dependency("Lz4_jll") Dependency("jemalloc_jll") From 1e6bc398ea24adf93997049719776898588bebbd Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Mon, 5 Sep 2022 12:03:26 +0200 Subject: [PATCH 05/22] Add Thrift as dep. --- A/Arrow/build_tarballs.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 1ab1883d162..f7c545e1d6e 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -46,13 +46,14 @@ products = [ ] # Dependencies that must be installed before this package can be built -dependencies = Dependency[ +dependencies = [ Dependency("boost_jll", compat="=1.76.0") Dependency("Zlib_jll") Dependency("Bzip2_jll", compat="1.0.7") Dependency("Zstd_jll") Dependency("Lz4_jll") Dependency("jemalloc_jll") + Dependency("Thrift_jll") ] # Build the tarballs, and possibly a `build.jl` as well. From a50ed96f3dbbf2ba7d20914590ea1030a21d5152 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Mon, 5 Sep 2022 22:07:48 +0200 Subject: [PATCH 06/22] Use only system libs. --- A/Arrow/build_tarballs.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index f7c545e1d6e..d0c9f0ba60d 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -19,6 +19,7 @@ CMAKE_FLAGS=(-DCMAKE_INSTALL_PREFIX=$prefix -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} -DCMAKE_BUILD_TYPE=Release -DARROW_BUILD_UTILITIES=ON +-DARROW_DEPENDENCY_SOURCE=SYSTEM -DARROW_BUILD_STATIC=OFF -DARROW_DATASET=ON -DARROW_WITH_BZ2=ON @@ -27,7 +28,8 @@ CMAKE_FLAGS=(-DCMAKE_INSTALL_PREFIX=$prefix -DARROW_WITH_ZLIB=ON -DARROW_WITH_SNAPPY=ON -DARROW_JEMALLOC_USE_SHARED=ON --DARROW_PARQUET=ON) +-DARROW_PARQUET=ON +) cmake . ${CMAKE_FLAGS[@]} @@ -54,7 +56,9 @@ dependencies = [ Dependency("Lz4_jll") Dependency("jemalloc_jll") Dependency("Thrift_jll") + Dependency("snappy_jll") ] # Build the tarballs, and possibly a `build.jl` as well. build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6") +b From 00ec791828bc0e2f4ac00b112e0ca689f267aba9 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Mon, 5 Sep 2022 23:15:00 +0200 Subject: [PATCH 07/22] Let's try a hack. --- A/Arrow/build_tarballs.jl | 27 +++++++++++++++++++++---- A/Arrow/bundled/patches/toolchain.patch | 14 +++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 A/Arrow/bundled/patches/toolchain.patch diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index d0c9f0ba60d..e57b6c61a1a 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -8,12 +8,20 @@ version = v"9.0.0" # Collection of sources required to complete build sources = [ ArchiveSource("https://github.com/apache/arrow/archive/refs/tags/apache-arrow-9.0.0.zip", "fb2469e9bfeb3e9a45f1e3086f536a329d59a0efd3c905436d3b6e8dd030be41") + DirectorySource("./bundled") ] # Bash recipe for building across all platforms script = raw""" -cd $WORKSPACE/srcdir/arrow-apache-arrow-*/cpp +cd $WORKSPACE/srcdir/arrow-apache-arrow-* + +# Set toolchain for building external deps +for f in ${WORKSPACE}/srcdir/patches/*.patch; do + atomic_patch -p1 ${f} +done + +cd cpp && mkdir build_dir && cd build_dir CMAKE_FLAGS=(-DCMAKE_INSTALL_PREFIX=$prefix -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} @@ -29,9 +37,20 @@ CMAKE_FLAGS=(-DCMAKE_INSTALL_PREFIX=$prefix -DARROW_WITH_SNAPPY=ON -DARROW_JEMALLOC_USE_SHARED=ON -DARROW_PARQUET=ON -) +-DARROW_SIMD_LEVEL=NONE +-DARROW_USE_XSIMD=OFF +-Dxsimd_SOURCE=AUTO +-Dre2_SOURCE=AUTO) + +echo $CMAKE_FLAGS + +# CMake is doubling the suffixes... +if [[ "${target}" == *-mingw32 ]]; then + ln -s ${prefix}/lib/libthrift.dll.a ${prefix}/lib/libthrift.a.dll.a + ln -s ${prefix}/lib/libutf8proc.a ${prefix}/lib/libutf8proc.dll.a.a +fi -cmake . ${CMAKE_FLAGS[@]} +cmake .. "${CMAKE_FLAGS[@]}" make -j${nproc} make install @@ -57,8 +76,8 @@ dependencies = [ Dependency("jemalloc_jll") Dependency("Thrift_jll") Dependency("snappy_jll") + Dependency("utf8proc_jll") ] # Build the tarballs, and possibly a `build.jl` as well. build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6") -b diff --git a/A/Arrow/bundled/patches/toolchain.patch b/A/Arrow/bundled/patches/toolchain.patch new file mode 100644 index 00000000000..c907849c6f7 --- /dev/null +++ b/A/Arrow/bundled/patches/toolchain.patch @@ -0,0 +1,14 @@ +diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake +index 5d1da18b7..61c907ad3 100644 +--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake +@@ -730,8 +730,7 @@ endif() + # directory. This leads to issues if the variables are exported in a subshell + # and the invocation of make/ninja is in distinct subshell without the same + # environment (CC/CXX). +-set(EP_COMMON_TOOLCHAIN -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} +- -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}) ++set(EP_COMMON_TOOLCHAIN -DCMAKE_TOOLCHAIN_FILE=$ENV{CMAKE_TARGET_TOOLCHAIN}) + + if(CMAKE_AR) + set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_AR=${CMAKE_AR}) From 26e042417faf12fd3c152e7b2eed90b9d1b56be1 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 11 Sep 2022 14:57:58 +0200 Subject: [PATCH 08/22] Use RE2_jll. --- A/Arrow/build_tarballs.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index e57b6c61a1a..51853785c61 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -77,6 +77,7 @@ dependencies = [ Dependency("Thrift_jll") Dependency("snappy_jll") Dependency("utf8proc_jll") + Dependency("RE2_jll") ] # Build the tarballs, and possibly a `build.jl` as well. From 3ea047d739620909b0e4c3e0fc203192d91a7659 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 11 Sep 2022 17:09:21 +0200 Subject: [PATCH 09/22] Use gcc v6 to get (partial) c++17. --- A/Arrow/build_tarballs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 51853785c61..25b4b176ca7 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -81,4 +81,4 @@ dependencies = [ ] # Build the tarballs, and possibly a `build.jl` as well. -build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6") +build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6", preferred_gcc_version=v"6") From bcb9b131be90a97333d83446f921b4ab75b09776 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 11 Sep 2022 17:42:38 +0200 Subject: [PATCH 10/22] Use gcc v8 to get full c++17 support. --- A/Arrow/build_tarballs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 25b4b176ca7..0d029b7bc39 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -81,4 +81,4 @@ dependencies = [ ] # Build the tarballs, and possibly a `build.jl` as well. -build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6", preferred_gcc_version=v"6") +build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6", preferred_gcc_version=v"8") From f222a831df5105503b778d0661985dc97c7ec1b2 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 11 Sep 2022 18:28:57 +0200 Subject: [PATCH 11/22] Disable building jemalloc and skip Bzip2 support when needed. --- A/Arrow/build_tarballs.jl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 0d029b7bc39..b96a95647ea 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -39,10 +39,17 @@ CMAKE_FLAGS=(-DCMAKE_INSTALL_PREFIX=$prefix -DARROW_PARQUET=ON -DARROW_SIMD_LEVEL=NONE -DARROW_USE_XSIMD=OFF +-DARROW_JEMALLOC=OFF -Dxsimd_SOURCE=AUTO -Dre2_SOURCE=AUTO) -echo $CMAKE_FLAGS +# Some platforms don't have BZ2 (yet) +if ! find ${libdir} -name "libhdf5*.${dlext}" -exec false '{}' +; then + CMAKE_FLAGS+=(-DARROW_WITH_BZ2=ON) +else + echo "Disabling BZ2 support" + CMAKE_FLAGS+=(-DARROW_WITH_BZ2=OFF) +fi # CMake is doubling the suffixes... if [[ "${target}" == *-mingw32 ]]; then From 75b9ba222c9f0ba2b82281b7cb8e3ee975ab18c7 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 30 Oct 2022 11:50:30 +0100 Subject: [PATCH 12/22] Simplify build considerably. --- A/Arrow/build_tarballs.jl | 19 +++++++++++++------ A/Arrow/bundled/patches/toolchain.patch | 12 ++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index b96a95647ea..2ab921170d8 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -26,17 +26,24 @@ cd cpp && mkdir build_dir && cd build_dir CMAKE_FLAGS=(-DCMAKE_INSTALL_PREFIX=$prefix -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} -DCMAKE_BUILD_TYPE=Release --DARROW_BUILD_UTILITIES=ON +-DARROW_BUILD_UTILITIES=OFF +-DARROW_WITH_UTF8PROC=OFF -DARROW_DEPENDENCY_SOURCE=SYSTEM +-DARROW_VERBOSE_THIRDPARTY_BUILD=ON -DARROW_BUILD_STATIC=OFF -DARROW_DATASET=ON +-DARROW_COMPUTE=OFF +-DARROW_WITH_RE2=OFF -DARROW_WITH_BZ2=ON +-DARROW_IPC=OFF -DARROW_WITH_LZ4=ON --DARROW_WITH_ZSTD=ON +-DARROW_WITH_ZSTD=OFF -DARROW_WITH_ZLIB=ON -DARROW_WITH_SNAPPY=ON -DARROW_JEMALLOC_USE_SHARED=ON +-DARROW_THRIFT_USE_SHARED=ON -DARROW_PARQUET=ON +-DPARQUET_BUILD_EXECUTABLES=OFF -DARROW_SIMD_LEVEL=NONE -DARROW_USE_XSIMD=OFF -DARROW_JEMALLOC=OFF @@ -78,13 +85,13 @@ dependencies = [ Dependency("boost_jll", compat="=1.76.0") Dependency("Zlib_jll") Dependency("Bzip2_jll", compat="1.0.7") - Dependency("Zstd_jll") + # Dependency("Zstd_jll") Dependency("Lz4_jll") - Dependency("jemalloc_jll") + # Dependency("jemalloc_jll") Dependency("Thrift_jll") Dependency("snappy_jll") - Dependency("utf8proc_jll") - Dependency("RE2_jll") + # Dependency("utf8proc_jll") + # Dependency("RE2_jll") ] # Build the tarballs, and possibly a `build.jl` as well. diff --git a/A/Arrow/bundled/patches/toolchain.patch b/A/Arrow/bundled/patches/toolchain.patch index c907849c6f7..30b524a764d 100644 --- a/A/Arrow/bundled/patches/toolchain.patch +++ b/A/Arrow/bundled/patches/toolchain.patch @@ -12,3 +12,15 @@ index 5d1da18b7..61c907ad3 100644 if(CMAKE_AR) set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_AR=${CMAKE_AR}) +diff --git a/cpp/cmake_modules/FindThrift.cmake b/cpp/cmake_modules/FindThrift.cmake +index 07028971d..8a937c426 100644 +--- a/cpp/cmake_modules/FindThrift.cmake ++++ b/cpp/cmake_modules/FindThrift.cmake +@@ -142,6 +142,7 @@ if(Thrift_FOUND) + endif() + set_target_properties(thrift::thrift + PROPERTIES IMPORTED_LOCATION "${THRIFT_LIB}" ++ IMPORTED_IMPLIB "${THRIFT_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}") + if(WIN32 AND NOT MSVC_TOOLCHAIN) + # We don't need this for Visual C++ because Thrift uses From a88724904933180e904803f786c7b2e026b5b213 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 30 Oct 2022 12:20:54 +0100 Subject: [PATCH 13/22] Upgrade to Arrow 10 to support ARM better. --- A/Arrow/build_tarballs.jl | 4 ++-- A/Arrow/bundled/patches/thrift.patch | 12 ++++++++++++ A/Arrow/bundled/patches/toolchain.patch | 16 ++-------------- 3 files changed, 16 insertions(+), 16 deletions(-) create mode 100644 A/Arrow/bundled/patches/thrift.patch diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 2ab921170d8..a816a7990e3 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -3,11 +3,11 @@ using BinaryBuilder, Pkg name = "Arrow" -version = v"9.0.0" +version = v"10.0.0" # Collection of sources required to complete build sources = [ - ArchiveSource("https://github.com/apache/arrow/archive/refs/tags/apache-arrow-9.0.0.zip", "fb2469e9bfeb3e9a45f1e3086f536a329d59a0efd3c905436d3b6e8dd030be41") + ArchiveSource("https://github.com/apache/arrow/archive/refs/tags/apache-arrow-10.0.0.zip", "42995abe620c41c42b8fbc486f1c63a1e5b1da534718ac66dbc790a88efeaa37") DirectorySource("./bundled") ] diff --git a/A/Arrow/bundled/patches/thrift.patch b/A/Arrow/bundled/patches/thrift.patch new file mode 100644 index 00000000000..a66201bbaf4 --- /dev/null +++ b/A/Arrow/bundled/patches/thrift.patch @@ -0,0 +1,12 @@ +diff --git a/cpp/cmake_modules/FindThrift.cmake b/cpp/cmake_modules/FindThrift.cmake +index 2f20a5cb5..2d1e728aa 100644 +--- a/cpp/cmake_modules/FindThrift.cmake ++++ b/cpp/cmake_modules/FindThrift.cmake +@@ -146,6 +146,7 @@ if(Thrift_FOUND) + endif() + set_target_properties(thrift::thrift + PROPERTIES IMPORTED_LOCATION "${THRIFT_LIB}" ++ IMPORTED_IMPLIB "${THRIFT_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}") + if(WIN32 AND NOT MSVC_TOOLCHAIN) + # We don't need this for Visual C++ because Thrift uses diff --git a/A/Arrow/bundled/patches/toolchain.patch b/A/Arrow/bundled/patches/toolchain.patch index 30b524a764d..33f3a559ac6 100644 --- a/A/Arrow/bundled/patches/toolchain.patch +++ b/A/Arrow/bundled/patches/toolchain.patch @@ -1,8 +1,8 @@ diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake -index 5d1da18b7..61c907ad3 100644 +index b7cd31f3d..676f9b93b 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake -@@ -730,8 +730,7 @@ endif() +@@ -753,8 +753,7 @@ endif() # directory. This leads to issues if the variables are exported in a subshell # and the invocation of make/ninja is in distinct subshell without the same # environment (CC/CXX). @@ -12,15 +12,3 @@ index 5d1da18b7..61c907ad3 100644 if(CMAKE_AR) set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_AR=${CMAKE_AR}) -diff --git a/cpp/cmake_modules/FindThrift.cmake b/cpp/cmake_modules/FindThrift.cmake -index 07028971d..8a937c426 100644 ---- a/cpp/cmake_modules/FindThrift.cmake -+++ b/cpp/cmake_modules/FindThrift.cmake -@@ -142,6 +142,7 @@ if(Thrift_FOUND) - endif() - set_target_properties(thrift::thrift - PROPERTIES IMPORTED_LOCATION "${THRIFT_LIB}" -+ IMPORTED_IMPLIB "${THRIFT_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}") - if(WIN32 AND NOT MSVC_TOOLCHAIN) - # We don't need this for Visual C++ because Thrift uses From 4aeaf24b8756f72759c425ca46366aefda0699af Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 30 Oct 2022 12:48:44 +0100 Subject: [PATCH 14/22] Patch support for armv6, disable older macos checks. --- A/Arrow/build_tarballs.jl | 5 +++++ A/Arrow/bundled/patches/cxxflags.patch | 15 +++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 A/Arrow/bundled/patches/cxxflags.patch diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index a816a7990e3..003f6f51d39 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -64,6 +64,11 @@ if [[ "${target}" == *-mingw32 ]]; then ln -s ${prefix}/lib/libutf8proc.a ${prefix}/lib/libutf8proc.dll.a.a fi +# Ignore check for availibility on older macOS versions +if [[ "${target}" == *-x86_64-apple-darwin ]]; then + CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" +fi + cmake .. "${CMAKE_FLAGS[@]}" make -j${nproc} diff --git a/A/Arrow/bundled/patches/cxxflags.patch b/A/Arrow/bundled/patches/cxxflags.patch new file mode 100644 index 00000000000..521792f67d6 --- /dev/null +++ b/A/Arrow/bundled/patches/cxxflags.patch @@ -0,0 +1,15 @@ +diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake +index cef4eb0b1..97f866938 100644 +--- a/cpp/cmake_modules/SetupCxxFlags.cmake ++++ b/cpp/cmake_modules/SetupCxxFlags.cmake +@@ -29,7 +29,9 @@ if(NOT DEFINED ARROW_CPU_FLAG) + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|ARM64|arm64") + set(ARROW_CPU_FLAG "armv8") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "armv7") +- set(ARROW_CPU_FLAG "armv7") ++ set(ARROW_CPU_FLAG "armv7") ++ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "armv6") ++ set(ARROW_CPU_FLAG "armv7") # Prevents another patch, will add -latomic to linker + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "powerpc|ppc") + set(ARROW_CPU_FLAG "ppc") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x") \ No newline at end of file From 2042ec84aedbb452d27f204e32a5f484cbce8700 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 30 Oct 2022 15:11:55 +0100 Subject: [PATCH 15/22] Fix passing CXX flags for macos. --- A/Arrow/build_tarballs.jl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 003f6f51d39..b0578e62439 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -23,8 +23,14 @@ done cd cpp && mkdir build_dir && cd build_dir +# Ignore check for availibility on older macOS versions +if [[ "${target}" == x86_64-apple-darwin* ]]; then + CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" +fi + CMAKE_FLAGS=(-DCMAKE_INSTALL_PREFIX=$prefix -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} +-DARROW_CXXFLAGS="${CXXFLAGS}" -DCMAKE_BUILD_TYPE=Release -DARROW_BUILD_UTILITIES=OFF -DARROW_WITH_UTF8PROC=OFF @@ -64,10 +70,6 @@ if [[ "${target}" == *-mingw32 ]]; then ln -s ${prefix}/lib/libutf8proc.a ${prefix}/lib/libutf8proc.dll.a.a fi -# Ignore check for availibility on older macOS versions -if [[ "${target}" == *-x86_64-apple-darwin ]]; then - CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" -fi cmake .. "${CMAKE_FLAGS[@]}" From e075b7286c2f3f7efe9323869d919307a6ae91d7 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 30 Oct 2022 15:32:05 +0100 Subject: [PATCH 16/22] Cleanup. --- A/Arrow/build_tarballs.jl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index b0578e62439..53f1219dba5 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -46,15 +46,13 @@ CMAKE_FLAGS=(-DCMAKE_INSTALL_PREFIX=$prefix -DARROW_WITH_ZSTD=OFF -DARROW_WITH_ZLIB=ON -DARROW_WITH_SNAPPY=ON --DARROW_JEMALLOC_USE_SHARED=ON -DARROW_THRIFT_USE_SHARED=ON -DARROW_PARQUET=ON -DPARQUET_BUILD_EXECUTABLES=OFF -DARROW_SIMD_LEVEL=NONE -DARROW_USE_XSIMD=OFF -DARROW_JEMALLOC=OFF --Dxsimd_SOURCE=AUTO --Dre2_SOURCE=AUTO) +-Dxsimd_SOURCE=AUTO) # Some platforms don't have BZ2 (yet) if ! find ${libdir} -name "libhdf5*.${dlext}" -exec false '{}' +; then @@ -92,13 +90,9 @@ dependencies = [ Dependency("boost_jll", compat="=1.76.0") Dependency("Zlib_jll") Dependency("Bzip2_jll", compat="1.0.7") - # Dependency("Zstd_jll") Dependency("Lz4_jll") - # Dependency("jemalloc_jll") Dependency("Thrift_jll") Dependency("snappy_jll") - # Dependency("utf8proc_jll") - # Dependency("RE2_jll") ] # Build the tarballs, and possibly a `build.jl` as well. From c6b2a8165b631575cf71b5094dfe539ba2b2e5a0 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 30 Oct 2022 20:49:31 +0100 Subject: [PATCH 17/22] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mosè Giordano --- A/Arrow/build_tarballs.jl | 3 ++- A/Arrow/bundled/patches/cxxflags.patch | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 53f1219dba5..01cf2ceb26a 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -7,7 +7,7 @@ version = v"10.0.0" # Collection of sources required to complete build sources = [ - ArchiveSource("https://github.com/apache/arrow/archive/refs/tags/apache-arrow-10.0.0.zip", "42995abe620c41c42b8fbc486f1c63a1e5b1da534718ac66dbc790a88efeaa37") + ArchiveSource("https://github.com/apache/arrow/archive/refs/tags/apache-arrow-$(version).zip", "42995abe620c41c42b8fbc486f1c63a1e5b1da534718ac66dbc790a88efeaa37") DirectorySource("./bundled") ] @@ -93,6 +93,7 @@ dependencies = [ Dependency("Lz4_jll") Dependency("Thrift_jll") Dependency("snappy_jll") + Dependency("CompilerSupportLibraries_jll") ] # Build the tarballs, and possibly a `build.jl` as well. diff --git a/A/Arrow/bundled/patches/cxxflags.patch b/A/Arrow/bundled/patches/cxxflags.patch index 521792f67d6..7d85334a5d9 100644 --- a/A/Arrow/bundled/patches/cxxflags.patch +++ b/A/Arrow/bundled/patches/cxxflags.patch @@ -12,4 +12,4 @@ index cef4eb0b1..97f866938 100644 + set(ARROW_CPU_FLAG "armv7") # Prevents another patch, will add -latomic to linker elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "powerpc|ppc") set(ARROW_CPU_FLAG "ppc") - elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x") \ No newline at end of file + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x") From 98b563cafdaa1c3da0e586a7a357ea4f18183f1d Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 30 Oct 2022 20:51:56 +0100 Subject: [PATCH 18/22] Remove temporary symlinks. Always build with BZ2. --- A/Arrow/build_tarballs.jl | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 01cf2ceb26a..084d9a8733e 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -54,24 +54,22 @@ CMAKE_FLAGS=(-DCMAKE_INSTALL_PREFIX=$prefix -DARROW_JEMALLOC=OFF -Dxsimd_SOURCE=AUTO) -# Some platforms don't have BZ2 (yet) -if ! find ${libdir} -name "libhdf5*.${dlext}" -exec false '{}' +; then - CMAKE_FLAGS+=(-DARROW_WITH_BZ2=ON) -else - echo "Disabling BZ2 support" - CMAKE_FLAGS+=(-DARROW_WITH_BZ2=OFF) -fi - # CMake is doubling the suffixes... if [[ "${target}" == *-mingw32 ]]; then ln -s ${prefix}/lib/libthrift.dll.a ${prefix}/lib/libthrift.a.dll.a ln -s ${prefix}/lib/libutf8proc.a ${prefix}/lib/libutf8proc.dll.a.a fi - cmake .. "${CMAKE_FLAGS[@]}" make -j${nproc} + +# Remove double suffixes +if [[ "${target}" == *-mingw32 ]]; then + rm ${prefix}/lib/libthrift.a.dll.a + rm ${prefix}/lib/libutf8proc.dll.a.a +fi + make install """ From 23f7375533a15a376634604a8eed913fa5774fc3 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Mon, 31 Oct 2022 08:18:07 +0100 Subject: [PATCH 19/22] Update A/Arrow/build_tarballs.jl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mosè Giordano --- A/Arrow/build_tarballs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 084d9a8733e..e68fc13fe64 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -87,7 +87,7 @@ products = [ dependencies = [ Dependency("boost_jll", compat="=1.76.0") Dependency("Zlib_jll") - Dependency("Bzip2_jll", compat="1.0.7") + Dependency("Bzip2_jll", compat="1.0.8") Dependency("Lz4_jll") Dependency("Thrift_jll") Dependency("snappy_jll") From eff59a011d1e50582a25a2bad6ce1d0644239957 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Wed, 2 Nov 2022 14:21:01 +0100 Subject: [PATCH 20/22] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mosè Giordano --- A/Arrow/build_tarballs.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index e68fc13fe64..066ebbe9e00 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -7,7 +7,8 @@ version = v"10.0.0" # Collection of sources required to complete build sources = [ - ArchiveSource("https://github.com/apache/arrow/archive/refs/tags/apache-arrow-$(version).zip", "42995abe620c41c42b8fbc486f1c63a1e5b1da534718ac66dbc790a88efeaa37") + ArchiveSource("https://github.com/apache/arrow/archive/refs/tags/apache-arrow-$(version).zip", + "42995abe620c41c42b8fbc486f1c63a1e5b1da534718ac66dbc790a88efeaa37") DirectorySource("./bundled") ] @@ -95,4 +96,5 @@ dependencies = [ ] # Build the tarballs, and possibly a `build.jl` as well. -build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6", preferred_gcc_version=v"8") +build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; + julia_compat="1.6", preferred_gcc_version=v"8") From c06381434001b985373465cbefe3c35b2a396407 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sun, 6 Nov 2022 13:14:24 +0100 Subject: [PATCH 21/22] Removed unneeded extension patching on mingw. --- A/Arrow/build_tarballs.jl | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 066ebbe9e00..926d1f6e1f9 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -55,22 +55,10 @@ CMAKE_FLAGS=(-DCMAKE_INSTALL_PREFIX=$prefix -DARROW_JEMALLOC=OFF -Dxsimd_SOURCE=AUTO) -# CMake is doubling the suffixes... -if [[ "${target}" == *-mingw32 ]]; then - ln -s ${prefix}/lib/libthrift.dll.a ${prefix}/lib/libthrift.a.dll.a - ln -s ${prefix}/lib/libutf8proc.a ${prefix}/lib/libutf8proc.dll.a.a -fi - cmake .. "${CMAKE_FLAGS[@]}" make -j${nproc} -# Remove double suffixes -if [[ "${target}" == *-mingw32 ]]; then - rm ${prefix}/lib/libthrift.a.dll.a - rm ${prefix}/lib/libutf8proc.dll.a.a -fi - make install """ From 0941ebf2cdcbde263d242e04247e800f18d36bc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Sun, 6 Nov 2022 19:39:27 +0000 Subject: [PATCH 22/22] Update A/Arrow/build_tarballs.jl --- A/Arrow/build_tarballs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/A/Arrow/build_tarballs.jl b/A/Arrow/build_tarballs.jl index 926d1f6e1f9..7a6bdec0cf3 100644 --- a/A/Arrow/build_tarballs.jl +++ b/A/Arrow/build_tarballs.jl @@ -80,7 +80,7 @@ dependencies = [ Dependency("Lz4_jll") Dependency("Thrift_jll") Dependency("snappy_jll") - Dependency("CompilerSupportLibraries_jll") + Dependency("CompilerSupportLibraries_jll"; platforms=filter(!Sys.isapple, platforms)) ] # Build the tarballs, and possibly a `build.jl` as well.