Merge remote-tracking branch 'upstream/master' into pytorch-macosx-arm64

.github/workflows/cpython.yml

@@ -21,10 +21,10 @@ jobs:
 #    runs-on: ubuntu-20.04
 #    steps:
 #      - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions
-#  linux-arm64:
-#    runs-on: ubuntu-20.04
-#    steps:
-#      - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions
+  linux-arm64:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions
 #  linux-ppc64le:
 #    runs-on: ubuntu-20.04
 #    steps:
@@ -50,7 +50,7 @@ jobs:
     steps:
       - uses: bytedeco/javacpp-presets/.github/actions/deploy-windows@actions
   redeploy:
-    needs: [linux-x86_64, macosx-x86_64, windows-x86_64]
+    needs: [linux-arm64, linux-x86_64, macosx-x86_64, windows-x86_64]
 #    needs: [linux-armhf, linux-arm64, linux-ppc64le, linux-x86, linux-x86_64, macosx-x86_64, windows-x86, windows-x86_64]
     runs-on: ubuntu-20.04
     steps:

.github/workflows/numpy.yml

@@ -22,10 +22,10 @@ jobs:
 #    runs-on: ubuntu-20.04
 #    steps:
 #      - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions
-#  linux-arm64:
-#    runs-on: ubuntu-20.04
-#    steps:
-#      - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions
+  linux-arm64:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions
 #  linux-ppc64le:
 #    runs-on: ubuntu-20.04
 #    steps:
@@ -51,7 +51,7 @@ jobs:
     steps:
       - uses: bytedeco/javacpp-presets/.github/actions/deploy-windows@actions
   redeploy:
-    needs: [linux-x86_64, macosx-x86_64, windows-x86_64]
+    needs: [linux-arm64, linux-x86_64, macosx-x86_64, windows-x86_64]
 #    needs: [linux-armhf, linux-arm64, linux-ppc64le, linux-x86, linux-x86_64, macosx-x86_64, windows-x86, windows-x86_64]
     runs-on: ubuntu-20.04
     steps:

CHANGELOG.md

@@ -1,13 +1,14 @@
 
  * Introduce `macosx-arm64` builds for PyTorch ([pull #1463](https://github.com/bytedeco/javacpp-presets/pull/1463))
+ * Reenable `linux-arm64` builds for CPython and NumPy ([pull #1386](https://github.com/bytedeco/javacpp-presets/pull/1386)
  * Enable Vulkan GPU acceleration for FFmpeg ([pull #1460](https://github.com/bytedeco/javacpp-presets/pull/1460))
  * Include `timeapi.h` for system API of Windows ([pull #1447](https://github.com/bytedeco/javacpp-presets/pull/1447))
  * Add Android and Windows builds to presets for DepthAI ([pull #1441](https://github.com/bytedeco/javacpp-presets/pull/1441))
  * Add presets for nvCOMP 3.0.5 ([pull #1434](https://github.com/bytedeco/javacpp-presets/pull/1434)), SentencePiece 0.1.99 ([pull #1384](https://github.com/bytedeco/javacpp-presets/pull/1384))
  * Refactor and improve presets for PyTorch ([pull #1360](https://github.com/bytedeco/javacpp-presets/pull/1360))
  * Include `mkl_lapack.h` header file in presets for MKL ([issue #1388](https://github.com/bytedeco/javacpp-presets/issues/1388))
  * Map new higher-level C++ API of Triton Inference Server ([pull #1361](https://github.com/bytedeco/javacpp-presets/pull/1361))
- * Upgrade presets for OpenCV 4.9.0, FFmpeg 6.1.1, HDF5 1.14.3, MKL 2024.0, DNNL 3.3.4, OpenBLAS 0.3.26, ARPACK-NG 3.9.1, CPython 3.12.1, NumPy 1.26.3, SciPy 1.11.4, LLVM 17.0.6, Leptonica 1.84.1, Tesseract 5.3.4, CUDA 12.3.2, cuDNN 8.9.7, NCCL 2.19.3, OpenCL 3.0.15, PyTorch 2.1.2 ([pull #1426](https://github.com/bytedeco/javacpp-presets/pull/1426)), TensorFlow Lite 2.15.0, Triton Inference Server 2.41.0, DepthAI 2.24.0, ONNX 1.15.0, ONNX Runtime 1.16.3, TVM 0.14.0, and their dependencies
+ * Upgrade presets for OpenCV 4.9.0, FFmpeg 6.1.1, HDF5 1.14.3, MKL 2024.0, DNNL 3.3.4, OpenBLAS 0.3.26, ARPACK-NG 3.9.1, CPython 3.12.1, NumPy 1.26.3, SciPy 1.12.0, LLVM 17.0.6, Leptonica 1.84.1, Tesseract 5.3.4, CUDA 12.3.2, cuDNN 8.9.7, NCCL 2.19.3, OpenCL 3.0.15, PyTorch 2.1.2 ([pull #1426](https://github.com/bytedeco/javacpp-presets/pull/1426)), TensorFlow Lite 2.15.0, Triton Inference Server 2.41.0, DepthAI 2.24.0, ONNX 1.15.0, ONNX Runtime 1.16.3, TVM 0.14.0, and their dependencies
 
 ### June 6, 2023 version 1.5.9
  * Virtualize `nvinfer1::IGpuAllocator` from TensorRT to allow customization ([pull #1367](https://github.com/bytedeco/javacpp-presets/pull/1367))

cpython/cppbuild.sh

@@ -24,6 +24,9 @@ patch -Np1 --binary < ../../../cpython-windows.patch
 
 case $PLATFORM in
     linux-armhf)
+        # armhf builds are no longer guaranteed to succeed.
+        # If they are needed, look at the configure command for arm64
+        # and try any options that are missing from here
         HOST_ARCH="$(uname -m)"
         CROSSCOMPILE=1
         if [[ $HOST_ARCH == *"arm"* ]]
@@ -49,12 +52,16 @@ case $PLATFORM in
         make install_sw
         make distclean
         cd ../Python-$CPYTHON_VERSION
-        ./configure --prefix=$INSTALL_PATH/host --with-system-ffi --with-openssl=$INSTALL_PATH/host
+        # ac_cv_buggy_getaddrinfo=no disables the runtime ./configure checks for ipv6 support
+        # Without it, the build fails because it can't detect ipv6 on the host. Needed on both host and cross-compiled builds
+        ./configure --prefix=$INSTALL_PATH/host --with-system-ffi --with-openssl=$INSTALL_PATH/host ac_cv_buggy_getaddrinfo=no
         make -j $MAKEJ
         make install
         make distclean
         export PATH=$INSTALL_PATH/host/bin/:$PATH
-        CC="arm-linux-gnueabihf-gcc -std=c99" ./configure --prefix=$INSTALL_PATH --host=arm-linux-gnueabihf --build=$(uname -m)-pc-linux-gnu --enable-shared --with-system-ffi --with-openssl=$INSTALL_PATH LDFLAGS='-s -Wl,-rpath,\$$ORIGIN/,-rpath,\$$ORIGIN/../,-rpath,\$$ORIGIN/../lib/' --with-build-python=$INSTALL_PATH/host/bin/python3 $INSTALL_PATH/host ac_cv_working_openssl_hashlib=yes ac_cv_working_openssl_ssl=yes
+        # ac_cv_file__dev_ptmx=yes and ac_cv_file__dev_ptc=no are required for cross-compilation as stated by the configure script,
+        # but little information is known about them
+        CC="arm-linux-gnueabihf-gcc -std=c99" ./configure --prefix=$INSTALL_PATH --host=arm-linux-gnueabihf --build=$(uname -m)-pc-linux-gnu --enable-shared --with-system-ffi --with-openssl=$INSTALL_PATH LDFLAGS='-s -Wl,-rpath,\$$ORIGIN/,-rpath,\$$ORIGIN/../,-rpath,\$$ORIGIN/../lib/' --with-build-python=$INSTALL_PATH/host/bin/python3 $INSTALL_PATH/host ac_cv_working_openssl_hashlib=yes ac_cv_working_openssl_ssl=yes ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptmx=yes ac_cv_file__dev_ptc=no
         make -j $MAKEJ
         make install
         ;;
@@ -70,12 +77,22 @@ case $PLATFORM in
         make install_sw
         make distclean
         cd ../Python-$CPYTHON_VERSION
-        ./configure --prefix=$INSTALL_PATH/host --with-system-ffi --with-openssl=$INSTALL_PATH/host
+        # ac_cv_buggy_getaddrinfo=no disables the runtime ./configure checks for ipv6 support
+        # Without it, the build fails because it can't detect ipv6 on the host. Needed on both host and cross-compiled builds
+        ./configure --prefix=$INSTALL_PATH/host --with-system-ffi --with-openssl=$INSTALL_PATH/host ac_cv_buggy_getaddrinfo=no
         make -j $MAKEJ
         make install
         make distclean
         export PATH=$INSTALL_PATH/host/bin/:$PATH
-        CC="aarch64-linux-gnu-gcc -mabi=lp64 $CFLAGS" ./configure --prefix=$INSTALL_PATH --host=aarch64-linux-gnu --build=$(uname -m)-pc-linux-gnu --enable-shared --with-system-ffi --with-openssl=$INSTALL_PATH LDFLAGS='-s -Wl,-rpath,\$$ORIGIN/,-rpath,\$$ORIGIN/../,-rpath,\$$ORIGIN/../lib/' --with-build-python=$INSTALL_PATH/host/bin/python3 ac_cv_working_openssl_hashlib=yes ac_cv_working_openssl_ssl=yes
+        # ac_cv_file__dev_ptmx=yes and ac_cv_file__dev_ptc=no are required for cross-compilation as stated by the configure script,
+        # but little information is known about them.
+        # /dev/ptmx is the pseudoterminal master file, reading from it generates a new file descriptor
+        # to use with a corresponding /dev/pts/ pseudoterminal
+        # See man 4 ptmx
+        # No information on /dev/ptc could be found.
+        # The above configure options specify whether the corresponding device files
+        # are expected to be found on the target machine.
+        CC="aarch64-linux-gnu-gcc -mabi=lp64 $CFLAGS" ./configure --prefix=$INSTALL_PATH --host=aarch64-linux-gnu --build=$(uname -m)-pc-linux-gnu --enable-shared --with-system-ffi --with-openssl=$INSTALL_PATH LDFLAGS='-s -Wl,-rpath,\$$ORIGIN/,-rpath,\$$ORIGIN/../,-rpath,\$$ORIGIN/../lib/' --with-build-python=$INSTALL_PATH/host/bin/python3 ac_cv_working_openssl_hashlib=yes ac_cv_working_openssl_ssl=yes ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptmx=yes ac_cv_file__dev_ptc=no
         make -j $MAKEJ
         make install
         ;;

cpython/platform/pom.xml

@@ -36,12 +36,12 @@
 <!--      <version>${project.version}</version>-->
 <!--      <classifier>${javacpp.platform.linux-armhf}</classifier>-->
 <!--    </dependency>-->
-<!--    <dependency>-->
-<!--      <groupId>${project.groupId}</groupId>-->
-<!--      <artifactId>${javacpp.moduleId}</artifactId>-->
-<!--      <version>${project.version}</version>-->
-<!--      <classifier>${javacpp.platform.linux-arm64}</classifier>-->
-<!--    </dependency>-->
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>${javacpp.moduleId}</artifactId>
+      <version>${project.version}</version>
+      <classifier>${javacpp.platform.linux-arm64}</classifier>
+    </dependency>
 <!--    <dependency>-->
 <!--      <groupId>${project.groupId}</groupId>-->
 <!--      <artifactId>${javacpp.moduleId}</artifactId>-->
@@ -136,7 +136,7 @@
                   <moduleInfoSource>
                     module org.bytedeco.${javacpp.moduleId}.platform {
 //                      requires static org.bytedeco.${javacpp.moduleId}.linux.armhf;
-//                      requires static org.bytedeco.${javacpp.moduleId}.linux.arm64;
+                      requires static org.bytedeco.${javacpp.moduleId}.linux.arm64;
 //                      requires static org.bytedeco.${javacpp.moduleId}.linux.ppc64le;
 //                      requires static org.bytedeco.${javacpp.moduleId}.linux.x86;
                       requires static org.bytedeco.${javacpp.moduleId}.linux.x86_64;

numpy/cppbuild.sh

@@ -102,6 +102,9 @@ if ! $PYTHON_BIN_PATH -m pip install --target=$PYTHON_LIB_PATH $TOOLS; then
     echo "extra_link_args = -lgfortran"           >> site.cfg
     chmod +x "$CPYTHON_HOST_PATH/bin/python3.12"
     export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$CPYTHON_HOST_PATH/lib/:$CPYTHON_HOST_PATH"
+
+    # crossenv 1.4 for python 3.11+ support.
+    # See https://github.com/bytedeco/javacpp-presets/issues/1381
     "$CPYTHON_HOST_PATH/bin/python3.12" -m pip install --target="$CPYTHON_HOST_PATH/lib/python3.12/" crossenv==1.4 $TOOLS
     "$CPYTHON_HOST_PATH/bin/python3.12" -m crossenv "$PYTHON_BIN_PATH" crossenv
     source crossenv/bin/activate
@@ -117,6 +120,8 @@ case $PLATFORM in
         arm-linux-gnueabihf-strip $(find ../ -iname *.so)
         ;;
     linux-arm64)
+        rm -f meson.build pyproject.toml
+        mv pyproject.toml.setuppy pyproject.toml
         ATLAS=None CC="aarch64-linux-gnu-gcc -mabi=lp64" CFLAGS="-O2" "$PYTHON_BIN_PATH" -m pip install . --prefix $INSTALL_PATH
         aarch64-linux-gnu-strip $(find ../ -iname *.so)
         ;;

numpy/platform/pom.xml

@@ -41,12 +41,12 @@
 <!--      <version>${project.version}</version>-->
 <!--      <classifier>${javacpp.platform.linux-armhf}</classifier>-->
 <!--    </dependency>-->
-<!--    <dependency>-->
-<!--      <groupId>${project.groupId}</groupId>-->
-<!--      <artifactId>${javacpp.moduleId}</artifactId>-->
-<!--      <version>${project.version}</version>-->
-<!--      <classifier>${javacpp.platform.linux-arm64}</classifier>-->
-<!--    </dependency>-->
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>${javacpp.moduleId}</artifactId>
+      <version>${project.version}</version>
+      <classifier>${javacpp.platform.linux-arm64}</classifier>
+    </dependency>
 <!--    <dependency>-->
 <!--      <groupId>${project.groupId}</groupId>-->
 <!--      <artifactId>${javacpp.moduleId}</artifactId>-->
@@ -141,7 +141,7 @@
                   <moduleInfoSource>
                     module org.bytedeco.${javacpp.moduleId}.platform {
 //                      requires static org.bytedeco.${javacpp.moduleId}.linux.armhf;
-//                      requires static org.bytedeco.${javacpp.moduleId}.linux.arm64;
+                      requires static org.bytedeco.${javacpp.moduleId}.linux.arm64;
 //                      requires static org.bytedeco.${javacpp.moduleId}.linux.ppc64le;
 //                      requires static org.bytedeco.${javacpp.moduleId}.linux.x86;
                       requires static org.bytedeco.${javacpp.moduleId}.linux.x86_64;

platform/pom.xml

@@ -222,7 +222,7 @@
     <dependency>
       <groupId>org.bytedeco</groupId>
       <artifactId>scipy-platform</artifactId>
-      <version>1.11.4-${project.version}</version>
+      <version>1.12.0-${project.version}</version>
     </dependency>
 <!--    <dependency>-->
 <!--      <groupId>org.bytedeco</groupId>-->

scipy/README.md

@@ -9,7 +9,7 @@ Introduction
 ------------
 This directory contains the JavaCPP Presets module for:
 
- * SciPy 1.11.4  https://www.scipy.org/
+ * SciPy 1.12.0  https://www.scipy.org/
 
 Please refer to the parent README.md file for more detailed information about the JavaCPP Presets.
 
@@ -48,7 +48,7 @@ We can use [Maven 3](http://maven.apache.org/) to download and install automatic
         <dependency>
             <groupId>org.bytedeco</groupId>
             <artifactId>scipy-platform</artifactId>
-            <version>1.11.4-1.5.10-SNAPSHOT</version>
+            <version>1.12.0-1.5.10-SNAPSHOT</version>
         </dependency>
 
         <!-- Additional dependencies to use bundled full version of MKL -->

scipy/cppbuild.sh

@@ -8,10 +8,11 @@ if [[ -z "$PLATFORM" ]]; then
 fi
 
 BOOST=1_75_0
-SCIPY_VERSION=1.11.4
+SCIPY_VERSION=1.12.0
 download http://downloads.sourceforge.net/project/boost/boost/${BOOST//_/.}/boost_$BOOST.tar.gz boost_$BOOST.tar.gz
+download https://github.com/data-apis/array-api-compat/archive/05548f0.tar.gz array-api-compat-05548f0.tar.gz
 download https://github.com/scipy/HiGHS/archive/4a12295.tar.gz HiGHS-4a12295.tar.gz
-download https://github.com/scipy/unuran/archive/81a1fd1.tar.gz unuran-81a1fd1.tar.gz
+download https://github.com/scipy/unuran/archive/1d315c6.tar.gz unuran-1d315c6.tar.gz
 download https://github.com/scipy/PROPACK/archive/96f6800.tar.gz PROPACK-96f6800.tar.gz
 download https://github.com/scipy/scipy/archive/v$SCIPY_VERSION.tar.gz scipy-$SCIPY_VERSION.tar.gz
 
@@ -51,11 +52,13 @@ NUMPY_PATH="${NUMPY_PATH//\\//}"
 
 echo "Decompressing archives..."
 tar --totals -xzf ../boost_$BOOST.tar.gz
+tar --totals -xzf ../array-api-compat-*.tar.gz
 tar --totals -xzf ../HiGHS-*.tar.gz
 tar --totals -xzf ../unuran-*.tar.gz
 tar --totals -xzf ../PROPACK-*.tar.gz
 tar --totals -xzf ../scipy-$SCIPY_VERSION.tar.gz
 cp -a boost_$BOOST/* scipy-$SCIPY_VERSION/scipy/_lib/boost_math/
+cp -a array-api-compat-*/* scipy-$SCIPY_VERSION/scipy/_lib/array_api_compat/
 cp -a HiGHS-*/* scipy-$SCIPY_VERSION/scipy/_lib/highs/
 cp -a unuran-*/* scipy-$SCIPY_VERSION/scipy/_lib/unuran/
 cp -a PROPACK-*/* scipy-$SCIPY_VERSION/scipy/sparse/linalg/_propack/PROPACK/
@@ -72,11 +75,11 @@ lapack = blas\\
 
 mkdir -p scipy/_lib/boost_math/include
 cp -a scipy/_lib/boost_math/boost scipy/_lib/boost_math/include
-mv _setup.py setup.py
+# mv _setup.py setup.py
 
 # prevent setuptools from trying to build NumPy
-sedinplace '/req_np/d' setup.py
-sedinplace 's/README.rst/README.md/g' scipy/_lib/setup.py
+# sedinplace '/req_np/d' setup.py
+# sedinplace 's/README.rst/README.md/g' scipy/_lib/setup.py
 
 echo "[openblas]"                                  > site.cfg
 echo "libraries = openblas"                       >> site.cfg

scipy/platform/pom.xml

@@ -12,7 +12,7 @@
 
   <groupId>org.bytedeco</groupId>
   <artifactId>scipy-platform</artifactId>
-  <version>1.11.4-${project.parent.version}</version>
+  <version>1.12.0-${project.parent.version}</version>
   <name>JavaCPP Presets Platform for SciPy</name>
 
   <properties>

scipy/pom.xml

@@ -11,7 +11,7 @@
 
   <groupId>org.bytedeco</groupId>
   <artifactId>scipy</artifactId>
-  <version>1.11.4-${project.parent.version}</version>
+  <version>1.12.0-${project.parent.version}</version>
   <name>JavaCPP Presets for SciPy</name>
 
   <dependencies>

scipy/samples/pom.xml

@@ -12,7 +12,7 @@
         <dependency>
             <groupId>org.bytedeco</groupId>
             <artifactId>scipy-platform</artifactId>
-            <version>1.11.4-1.5.10-SNAPSHOT</version>
+            <version>1.12.0-1.5.10-SNAPSHOT</version>
         </dependency>
 
         <!-- Additional dependencies to use bundled full version of MKL -->

tvm/platform/gpu/pom.xml

@@ -39,7 +39,7 @@
     <dependency>
       <groupId>org.bytedeco</groupId>
       <artifactId>scipy-platform</artifactId>
-      <version>1.11.4-${project.parent.version}</version>
+      <version>1.12.0-${project.parent.version}</version>
     </dependency>
     <dependency>
       <groupId>org.bytedeco</groupId>

tvm/platform/pom.xml

@@ -38,7 +38,7 @@
     <dependency>
       <groupId>org.bytedeco</groupId>
       <artifactId>scipy-platform</artifactId>
-      <version>1.11.4-${project.parent.version}</version>
+      <version>1.12.0-${project.parent.version}</version>
     </dependency>
     <dependency>
       <groupId>org.bytedeco</groupId>

tvm/pom.xml

@@ -33,7 +33,7 @@
     <dependency>
       <groupId>org.bytedeco</groupId>
       <artifactId>scipy</artifactId>
-      <version>1.11.4-${project.parent.version}</version>
+      <version>1.12.0-${project.parent.version}</version>
     </dependency>
     <dependency>
       <groupId>org.bytedeco</groupId>
@@ -71,7 +71,7 @@
           <dependency>
             <groupId>org.bytedeco</groupId>
             <artifactId>scipy-platform</artifactId>
-            <version>1.11.4-${project.parent.version}</version>
+            <version>1.12.0-${project.parent.version}</version>
           </dependency>
         </dependencies>
         <configuration>