From ec319c565cb665be6f0e748a469ed9499160cee9 Mon Sep 17 00:00:00 2001 From: Samuel Audet Date: Tue, 23 Jan 2024 19:10:26 +0900 Subject: [PATCH 1/2] * Upgrade presets for SciPy 1.12.0 --- CHANGELOG.md | 2 +- platform/pom.xml | 2 +- scipy/README.md | 4 ++-- scipy/cppbuild.sh | 13 ++++++++----- scipy/platform/pom.xml | 2 +- scipy/pom.xml | 2 +- scipy/samples/pom.xml | 2 +- tvm/platform/gpu/pom.xml | 2 +- tvm/platform/pom.xml | 2 +- tvm/pom.xml | 4 ++-- 10 files changed, 19 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d6174fc5f0..5725807718e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ * Refactor and improve presets for PyTorch ([pull #1360](https://github.com/bytedeco/javacpp-presets/pull/1360)) * Include `mkl_lapack.h` header file in presets for MKL ([issue #1388](https://github.com/bytedeco/javacpp-presets/issues/1388)) * Map new higher-level C++ API of Triton Inference Server ([pull #1361](https://github.com/bytedeco/javacpp-presets/pull/1361)) - * Upgrade presets for OpenCV 4.9.0, FFmpeg 6.1.1, HDF5 1.14.3, MKL 2024.0, DNNL 3.3.4, OpenBLAS 0.3.26, ARPACK-NG 3.9.1, CPython 3.12.1, NumPy 1.26.3, SciPy 1.11.4, LLVM 17.0.6, Leptonica 1.84.1, Tesseract 5.3.4, CUDA 12.3.2, cuDNN 8.9.7, NCCL 2.19.3, OpenCL 3.0.15, PyTorch 2.1.2 ([pull #1426](https://github.com/bytedeco/javacpp-presets/pull/1426)), TensorFlow Lite 2.15.0, Triton Inference Server 2.41.0, DepthAI 2.24.0, ONNX 1.15.0, ONNX Runtime 1.16.3, TVM 0.14.0, and their dependencies + * Upgrade presets for OpenCV 4.9.0, FFmpeg 6.1.1, HDF5 1.14.3, MKL 2024.0, DNNL 3.3.4, OpenBLAS 0.3.26, ARPACK-NG 3.9.1, CPython 3.12.1, NumPy 1.26.3, SciPy 1.12.0, LLVM 17.0.6, Leptonica 1.84.1, Tesseract 5.3.4, CUDA 12.3.2, cuDNN 8.9.7, NCCL 2.19.3, OpenCL 3.0.15, PyTorch 2.1.2 ([pull #1426](https://github.com/bytedeco/javacpp-presets/pull/1426)), TensorFlow Lite 2.15.0, Triton Inference Server 2.41.0, DepthAI 2.24.0, ONNX 1.15.0, ONNX Runtime 1.16.3, TVM 0.14.0, and their dependencies ### June 6, 2023 version 1.5.9 * Virtualize `nvinfer1::IGpuAllocator` from TensorRT to allow customization ([pull #1367](https://github.com/bytedeco/javacpp-presets/pull/1367)) diff --git a/platform/pom.xml b/platform/pom.xml index 45d518d7513..80fb98c6ef7 100644 --- a/platform/pom.xml +++ b/platform/pom.xml @@ -222,7 +222,7 @@ org.bytedeco scipy-platform - 1.11.4-${project.version} + 1.12.0-${project.version} diff --git a/scipy/README.md b/scipy/README.md index f9642300f75..2f5f5dc1f9d 100644 --- a/scipy/README.md +++ b/scipy/README.md @@ -9,7 +9,7 @@ Introduction ------------ This directory contains the JavaCPP Presets module for: - * SciPy 1.11.4 https://www.scipy.org/ + * SciPy 1.12.0 https://www.scipy.org/ Please refer to the parent README.md file for more detailed information about the JavaCPP Presets. @@ -48,7 +48,7 @@ We can use [Maven 3](http://maven.apache.org/) to download and install automatic org.bytedeco scipy-platform - 1.11.4-1.5.10-SNAPSHOT + 1.12.0-1.5.10-SNAPSHOT diff --git a/scipy/cppbuild.sh b/scipy/cppbuild.sh index 57c7202f503..f00162b1705 100755 --- a/scipy/cppbuild.sh +++ b/scipy/cppbuild.sh @@ -8,10 +8,11 @@ if [[ -z "$PLATFORM" ]]; then fi BOOST=1_75_0 -SCIPY_VERSION=1.11.4 +SCIPY_VERSION=1.12.0 download http://downloads.sourceforge.net/project/boost/boost/${BOOST//_/.}/boost_$BOOST.tar.gz boost_$BOOST.tar.gz +download https://github.com/data-apis/array-api-compat/archive/05548f0.tar.gz array-api-compat-05548f0.tar.gz download https://github.com/scipy/HiGHS/archive/4a12295.tar.gz HiGHS-4a12295.tar.gz -download https://github.com/scipy/unuran/archive/81a1fd1.tar.gz unuran-81a1fd1.tar.gz +download https://github.com/scipy/unuran/archive/1d315c6.tar.gz unuran-1d315c6.tar.gz download https://github.com/scipy/PROPACK/archive/96f6800.tar.gz PROPACK-96f6800.tar.gz download https://github.com/scipy/scipy/archive/v$SCIPY_VERSION.tar.gz scipy-$SCIPY_VERSION.tar.gz @@ -51,11 +52,13 @@ NUMPY_PATH="${NUMPY_PATH//\\//}" echo "Decompressing archives..." tar --totals -xzf ../boost_$BOOST.tar.gz +tar --totals -xzf ../array-api-compat-*.tar.gz tar --totals -xzf ../HiGHS-*.tar.gz tar --totals -xzf ../unuran-*.tar.gz tar --totals -xzf ../PROPACK-*.tar.gz tar --totals -xzf ../scipy-$SCIPY_VERSION.tar.gz cp -a boost_$BOOST/* scipy-$SCIPY_VERSION/scipy/_lib/boost_math/ +cp -a array-api-compat-*/* scipy-$SCIPY_VERSION/scipy/_lib/array_api_compat/ cp -a HiGHS-*/* scipy-$SCIPY_VERSION/scipy/_lib/highs/ cp -a unuran-*/* scipy-$SCIPY_VERSION/scipy/_lib/unuran/ cp -a PROPACK-*/* scipy-$SCIPY_VERSION/scipy/sparse/linalg/_propack/PROPACK/ @@ -72,11 +75,11 @@ lapack = blas\\ mkdir -p scipy/_lib/boost_math/include cp -a scipy/_lib/boost_math/boost scipy/_lib/boost_math/include -mv _setup.py setup.py +# mv _setup.py setup.py # prevent setuptools from trying to build NumPy -sedinplace '/req_np/d' setup.py -sedinplace 's/README.rst/README.md/g' scipy/_lib/setup.py +# sedinplace '/req_np/d' setup.py +# sedinplace 's/README.rst/README.md/g' scipy/_lib/setup.py echo "[openblas]" > site.cfg echo "libraries = openblas" >> site.cfg diff --git a/scipy/platform/pom.xml b/scipy/platform/pom.xml index 7636893417e..2b613215c5a 100644 --- a/scipy/platform/pom.xml +++ b/scipy/platform/pom.xml @@ -12,7 +12,7 @@ org.bytedeco scipy-platform - 1.11.4-${project.parent.version} + 1.12.0-${project.parent.version} JavaCPP Presets Platform for SciPy diff --git a/scipy/pom.xml b/scipy/pom.xml index f17fe76eeda..92c9b4789e7 100644 --- a/scipy/pom.xml +++ b/scipy/pom.xml @@ -11,7 +11,7 @@ org.bytedeco scipy - 1.11.4-${project.parent.version} + 1.12.0-${project.parent.version} JavaCPP Presets for SciPy diff --git a/scipy/samples/pom.xml b/scipy/samples/pom.xml index 9a6a0c59a3d..98e36301b48 100644 --- a/scipy/samples/pom.xml +++ b/scipy/samples/pom.xml @@ -12,7 +12,7 @@ org.bytedeco scipy-platform - 1.11.4-1.5.10-SNAPSHOT + 1.12.0-1.5.10-SNAPSHOT diff --git a/tvm/platform/gpu/pom.xml b/tvm/platform/gpu/pom.xml index 3aeb8af77ae..dea05f1f5cf 100644 --- a/tvm/platform/gpu/pom.xml +++ b/tvm/platform/gpu/pom.xml @@ -39,7 +39,7 @@ org.bytedeco scipy-platform - 1.11.4-${project.parent.version} + 1.12.0-${project.parent.version} org.bytedeco diff --git a/tvm/platform/pom.xml b/tvm/platform/pom.xml index 9b93a2086a7..7fc2bab09fd 100644 --- a/tvm/platform/pom.xml +++ b/tvm/platform/pom.xml @@ -38,7 +38,7 @@ org.bytedeco scipy-platform - 1.11.4-${project.parent.version} + 1.12.0-${project.parent.version} org.bytedeco diff --git a/tvm/pom.xml b/tvm/pom.xml index 5979da7685c..2fcf6b44c8b 100644 --- a/tvm/pom.xml +++ b/tvm/pom.xml @@ -33,7 +33,7 @@ org.bytedeco scipy - 1.11.4-${project.parent.version} + 1.12.0-${project.parent.version} org.bytedeco @@ -71,7 +71,7 @@ org.bytedeco scipy-platform - 1.11.4-${project.parent.version} + 1.12.0-${project.parent.version} From bfdd4943b248dc8df1e56c240bacc4640da06b0d Mon Sep 17 00:00:00 2001 From: Branden Butler Date: Tue, 23 Jan 2024 20:13:37 -0600 Subject: [PATCH 2/2] * Reenable `linux-arm64` builds for CPython and NumPy (pull #1386) --- .github/workflows/cpython.yml | 10 +++++----- .github/workflows/numpy.yml | 10 +++++----- CHANGELOG.md | 1 + cpython/cppbuild.sh | 25 +++++++++++++++++++++---- cpython/platform/pom.xml | 14 +++++++------- numpy/cppbuild.sh | 5 +++++ numpy/platform/pom.xml | 14 +++++++------- 7 files changed, 51 insertions(+), 28 deletions(-) diff --git a/.github/workflows/cpython.yml b/.github/workflows/cpython.yml index ca3961363ce..94a400cc737 100644 --- a/.github/workflows/cpython.yml +++ b/.github/workflows/cpython.yml @@ -21,10 +21,10 @@ jobs: # runs-on: ubuntu-20.04 # steps: # - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions -# linux-arm64: -# runs-on: ubuntu-20.04 -# steps: -# - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions + linux-arm64: + runs-on: ubuntu-20.04 + steps: + - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions # linux-ppc64le: # runs-on: ubuntu-20.04 # steps: @@ -50,7 +50,7 @@ jobs: steps: - uses: bytedeco/javacpp-presets/.github/actions/deploy-windows@actions redeploy: - needs: [linux-x86_64, macosx-x86_64, windows-x86_64] + needs: [linux-arm64, linux-x86_64, macosx-x86_64, windows-x86_64] # needs: [linux-armhf, linux-arm64, linux-ppc64le, linux-x86, linux-x86_64, macosx-x86_64, windows-x86, windows-x86_64] runs-on: ubuntu-20.04 steps: diff --git a/.github/workflows/numpy.yml b/.github/workflows/numpy.yml index 4fbf0144f99..2a9e5b9989e 100644 --- a/.github/workflows/numpy.yml +++ b/.github/workflows/numpy.yml @@ -22,10 +22,10 @@ jobs: # runs-on: ubuntu-20.04 # steps: # - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions -# linux-arm64: -# runs-on: ubuntu-20.04 -# steps: -# - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions + linux-arm64: + runs-on: ubuntu-20.04 + steps: + - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions # linux-ppc64le: # runs-on: ubuntu-20.04 # steps: @@ -51,7 +51,7 @@ jobs: steps: - uses: bytedeco/javacpp-presets/.github/actions/deploy-windows@actions redeploy: - needs: [linux-x86_64, macosx-x86_64, windows-x86_64] + needs: [linux-arm64, linux-x86_64, macosx-x86_64, windows-x86_64] # needs: [linux-armhf, linux-arm64, linux-ppc64le, linux-x86, linux-x86_64, macosx-x86_64, windows-x86, windows-x86_64] runs-on: ubuntu-20.04 steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 5725807718e..141840253ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ + * Reenable `linux-arm64` builds for CPython and NumPy ([pull #1386](https://github.com/bytedeco/javacpp-presets/pull/1386) * Enable Vulkan GPU acceleration for FFmpeg ([pull #1460](https://github.com/bytedeco/javacpp-presets/pull/1460)) * Include `timeapi.h` for system API of Windows ([pull #1447](https://github.com/bytedeco/javacpp-presets/pull/1447)) * Add Android and Windows builds to presets for DepthAI ([pull #1441](https://github.com/bytedeco/javacpp-presets/pull/1441)) diff --git a/cpython/cppbuild.sh b/cpython/cppbuild.sh index 02c2ae6f438..310f01e883d 100755 --- a/cpython/cppbuild.sh +++ b/cpython/cppbuild.sh @@ -24,6 +24,9 @@ patch -Np1 --binary < ../../../cpython-windows.patch case $PLATFORM in linux-armhf) + # armhf builds are no longer guaranteed to succeed. + # If they are needed, look at the configure command for arm64 + # and try any options that are missing from here HOST_ARCH="$(uname -m)" CROSSCOMPILE=1 if [[ $HOST_ARCH == *"arm"* ]] @@ -49,12 +52,16 @@ case $PLATFORM in make install_sw make distclean cd ../Python-$CPYTHON_VERSION - ./configure --prefix=$INSTALL_PATH/host --with-system-ffi --with-openssl=$INSTALL_PATH/host + # ac_cv_buggy_getaddrinfo=no disables the runtime ./configure checks for ipv6 support + # Without it, the build fails because it can't detect ipv6 on the host. Needed on both host and cross-compiled builds + ./configure --prefix=$INSTALL_PATH/host --with-system-ffi --with-openssl=$INSTALL_PATH/host ac_cv_buggy_getaddrinfo=no make -j $MAKEJ make install make distclean export PATH=$INSTALL_PATH/host/bin/:$PATH - CC="arm-linux-gnueabihf-gcc -std=c99" ./configure --prefix=$INSTALL_PATH --host=arm-linux-gnueabihf --build=$(uname -m)-pc-linux-gnu --enable-shared --with-system-ffi --with-openssl=$INSTALL_PATH LDFLAGS='-s -Wl,-rpath,\$$ORIGIN/,-rpath,\$$ORIGIN/../,-rpath,\$$ORIGIN/../lib/' --with-build-python=$INSTALL_PATH/host/bin/python3 $INSTALL_PATH/host ac_cv_working_openssl_hashlib=yes ac_cv_working_openssl_ssl=yes + # ac_cv_file__dev_ptmx=yes and ac_cv_file__dev_ptc=no are required for cross-compilation as stated by the configure script, + # but little information is known about them + CC="arm-linux-gnueabihf-gcc -std=c99" ./configure --prefix=$INSTALL_PATH --host=arm-linux-gnueabihf --build=$(uname -m)-pc-linux-gnu --enable-shared --with-system-ffi --with-openssl=$INSTALL_PATH LDFLAGS='-s -Wl,-rpath,\$$ORIGIN/,-rpath,\$$ORIGIN/../,-rpath,\$$ORIGIN/../lib/' --with-build-python=$INSTALL_PATH/host/bin/python3 $INSTALL_PATH/host ac_cv_working_openssl_hashlib=yes ac_cv_working_openssl_ssl=yes ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptmx=yes ac_cv_file__dev_ptc=no make -j $MAKEJ make install ;; @@ -70,12 +77,22 @@ case $PLATFORM in make install_sw make distclean cd ../Python-$CPYTHON_VERSION - ./configure --prefix=$INSTALL_PATH/host --with-system-ffi --with-openssl=$INSTALL_PATH/host + # ac_cv_buggy_getaddrinfo=no disables the runtime ./configure checks for ipv6 support + # Without it, the build fails because it can't detect ipv6 on the host. Needed on both host and cross-compiled builds + ./configure --prefix=$INSTALL_PATH/host --with-system-ffi --with-openssl=$INSTALL_PATH/host ac_cv_buggy_getaddrinfo=no make -j $MAKEJ make install make distclean export PATH=$INSTALL_PATH/host/bin/:$PATH - CC="aarch64-linux-gnu-gcc -mabi=lp64 $CFLAGS" ./configure --prefix=$INSTALL_PATH --host=aarch64-linux-gnu --build=$(uname -m)-pc-linux-gnu --enable-shared --with-system-ffi --with-openssl=$INSTALL_PATH LDFLAGS='-s -Wl,-rpath,\$$ORIGIN/,-rpath,\$$ORIGIN/../,-rpath,\$$ORIGIN/../lib/' --with-build-python=$INSTALL_PATH/host/bin/python3 ac_cv_working_openssl_hashlib=yes ac_cv_working_openssl_ssl=yes + # ac_cv_file__dev_ptmx=yes and ac_cv_file__dev_ptc=no are required for cross-compilation as stated by the configure script, + # but little information is known about them. + # /dev/ptmx is the pseudoterminal master file, reading from it generates a new file descriptor + # to use with a corresponding /dev/pts/ pseudoterminal + # See man 4 ptmx + # No information on /dev/ptc could be found. + # The above configure options specify whether the corresponding device files + # are expected to be found on the target machine. + CC="aarch64-linux-gnu-gcc -mabi=lp64 $CFLAGS" ./configure --prefix=$INSTALL_PATH --host=aarch64-linux-gnu --build=$(uname -m)-pc-linux-gnu --enable-shared --with-system-ffi --with-openssl=$INSTALL_PATH LDFLAGS='-s -Wl,-rpath,\$$ORIGIN/,-rpath,\$$ORIGIN/../,-rpath,\$$ORIGIN/../lib/' --with-build-python=$INSTALL_PATH/host/bin/python3 ac_cv_working_openssl_hashlib=yes ac_cv_working_openssl_ssl=yes ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptmx=yes ac_cv_file__dev_ptc=no make -j $MAKEJ make install ;; diff --git a/cpython/platform/pom.xml b/cpython/platform/pom.xml index fcb3f29a51c..5f38cb158ec 100644 --- a/cpython/platform/pom.xml +++ b/cpython/platform/pom.xml @@ -36,12 +36,12 @@ - - - - - - + + ${project.groupId} + ${javacpp.moduleId} + ${project.version} + ${javacpp.platform.linux-arm64} + @@ -136,7 +136,7 @@ module org.bytedeco.${javacpp.moduleId}.platform { // requires static org.bytedeco.${javacpp.moduleId}.linux.armhf; -// requires static org.bytedeco.${javacpp.moduleId}.linux.arm64; + requires static org.bytedeco.${javacpp.moduleId}.linux.arm64; // requires static org.bytedeco.${javacpp.moduleId}.linux.ppc64le; // requires static org.bytedeco.${javacpp.moduleId}.linux.x86; requires static org.bytedeco.${javacpp.moduleId}.linux.x86_64; diff --git a/numpy/cppbuild.sh b/numpy/cppbuild.sh index cdfb1943309..7d8c25b5a72 100755 --- a/numpy/cppbuild.sh +++ b/numpy/cppbuild.sh @@ -102,6 +102,9 @@ if ! $PYTHON_BIN_PATH -m pip install --target=$PYTHON_LIB_PATH $TOOLS; then echo "extra_link_args = -lgfortran" >> site.cfg chmod +x "$CPYTHON_HOST_PATH/bin/python3.12" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$CPYTHON_HOST_PATH/lib/:$CPYTHON_HOST_PATH" + + # crossenv 1.4 for python 3.11+ support. + # See https://github.com/bytedeco/javacpp-presets/issues/1381 "$CPYTHON_HOST_PATH/bin/python3.12" -m pip install --target="$CPYTHON_HOST_PATH/lib/python3.12/" crossenv==1.4 $TOOLS "$CPYTHON_HOST_PATH/bin/python3.12" -m crossenv "$PYTHON_BIN_PATH" crossenv source crossenv/bin/activate @@ -117,6 +120,8 @@ case $PLATFORM in arm-linux-gnueabihf-strip $(find ../ -iname *.so) ;; linux-arm64) + rm -f meson.build pyproject.toml + mv pyproject.toml.setuppy pyproject.toml ATLAS=None CC="aarch64-linux-gnu-gcc -mabi=lp64" CFLAGS="-O2" "$PYTHON_BIN_PATH" -m pip install . --prefix $INSTALL_PATH aarch64-linux-gnu-strip $(find ../ -iname *.so) ;; diff --git a/numpy/platform/pom.xml b/numpy/platform/pom.xml index 3df444464b9..dd9e5c9083b 100644 --- a/numpy/platform/pom.xml +++ b/numpy/platform/pom.xml @@ -41,12 +41,12 @@ - - - - - - + + ${project.groupId} + ${javacpp.moduleId} + ${project.version} + ${javacpp.platform.linux-arm64} + @@ -141,7 +141,7 @@ module org.bytedeco.${javacpp.moduleId}.platform { // requires static org.bytedeco.${javacpp.moduleId}.linux.armhf; -// requires static org.bytedeco.${javacpp.moduleId}.linux.arm64; + requires static org.bytedeco.${javacpp.moduleId}.linux.arm64; // requires static org.bytedeco.${javacpp.moduleId}.linux.ppc64le; // requires static org.bytedeco.${javacpp.moduleId}.linux.x86; requires static org.bytedeco.${javacpp.moduleId}.linux.x86_64;