Merge branch 'main' of https://github.com/R-JunmingChen/arrow into AR…

…ROW-36420

.asf.yaml

@@ -24,6 +24,7 @@ github:
     - danepitkin
     - davisusanibar
     - felipecrv
+    - js8544
     - mapleFU
 
 notifications:

.env

@@ -98,7 +98,7 @@ VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1"    # 2023.04.15 Release
 # ci/docker/python-wheel-windows-vs2017.dockerfile.
 # This is a workaround for our CI problem that "archery docker build" doesn't
 # use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2022-06-12
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2023-08-02
 
 # Use conanio/${CONAN} for "docker-compose run --rm conan". See
 # https://github.com/conan-io/conan-docker-tools#readme for available

.github/CODEOWNERS

@@ -43,7 +43,6 @@
 /java/ @lidavidm
 /js/ @domoritz @trxcllnt
 /matlab/ @kevingurney @kou
-/python/ @AlenkaF
 /python/pyarrow/_flight.pyx @lidavidm
 /python/pyarrow/**/*gandiva* @wjones127
 /r/ @paleolimbot @thisisnic

.github/dependabot.yml

@@ -23,6 +23,12 @@ updates:
       interval: "weekly"
     commit-message:
       prefix: "MINOR: [CI] "
+  - package-ecosystem: "npm"
+    directory: "/js/"
+    schedule:
+      interval: "monthly"
+    commit-message:
+      prefix: "MINOR: [JS] "
   - package-ecosystem: "nuget"
     directory: "/csharp/"
     schedule:

.github/workflows/cpp.yml

@@ -23,6 +23,7 @@ on:
       - '.github/workflows/cpp.yml'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
+      - 'ci/scripts/install_azurite.sh'
       - 'ci/scripts/install_gcs_testbench.sh'
       - 'ci/scripts/install_minio.sh'
       - 'ci/scripts/msys2_*'
@@ -34,6 +35,7 @@ on:
       - '.github/workflows/cpp.yml'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
+      - 'ci/scripts/install_azurite.sh'
       - 'ci/scripts/install_gcs_testbench.sh'
       - 'ci/scripts/install_minio.sh'
       - 'ci/scripts/msys2_*'
@@ -67,7 +69,8 @@ jobs:
             image: conda-cpp
             llvm: "14"
             runs-on: ubuntu-latest
-            title: AMD64 Conda C++
+            simd-level: AVX2
+            title: AMD64 Conda C++ AVX2
             ubuntu: "22.04"
           - arch: amd64
             clang-tools: "14"
@@ -85,6 +88,7 @@ jobs:
             ubuntu: "20.04"
     env:
       ARCH: ${{ matrix.arch }}
+      ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
       CLANG_TOOLS: ${{ matrix.clang-tools }}
       LLVM: ${{ matrix.llvm }}
       UBUNTU: ${{ matrix.ubuntu }}
@@ -151,6 +155,7 @@ jobs:
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 75
     env:
+      ARROW_AZURE: ON
       ARROW_BUILD_TESTS: ON
       ARROW_DATASET: ON
       ARROW_FLIGHT: ON
@@ -159,8 +164,7 @@ jobs:
       ARROW_HDFS: ON
       ARROW_HOME: /usr/local
       ARROW_JEMALLOC: ON
-      # TODO(kszucs): link error in the tests
-      ARROW_ORC: OFF
+      ARROW_ORC: ON
       ARROW_PARQUET: ON
       ARROW_S3: ON
       ARROW_WITH_BROTLI: ON
@@ -174,6 +178,10 @@ jobs:
       ARROW_WITH_ZSTD: ON
       GTest_SOURCE: BUNDLED
     steps:
+      - name: CPU Info
+        run: |
+          sysctl -a | grep cpu
+          sysctl -a | grep "hw.optional"
       - name: Checkout Arrow
         uses: actions/checkout@v3
         with:
@@ -194,6 +202,8 @@ jobs:
             ci/scripts/install_minio.sh latest /usr/local
       - name: Install Google Cloud Storage Testbench
         run: ci/scripts/install_gcs_testbench.sh default
+      - name: Install Azurite Storage Emulator
+        run: ci/scripts/install_azurite.sh
       - name: Setup ccache
         run: |
           ci/scripts/ccache_setup.sh
@@ -219,7 +229,7 @@ jobs:
           ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
 
   windows:
-    name: AMD64 ${{ matrix.name }} C++17
+    name: ${{ matrix.title }}
     runs-on: ${{ matrix.os }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 60
@@ -230,7 +240,8 @@ jobs:
           - windows-2019
         include:
           - os: windows-2019
-            name: Windows 2019
+            simd-level: AVX2
+            title: AMD64 Windows 2019 C++17 AVX2
     env:
       ARROW_BOOST_USE_SHARED: OFF
       ARROW_BUILD_BENCHMARKS: ON
@@ -245,6 +256,7 @@ jobs:
       ARROW_MIMALLOC: ON
       ARROW_ORC: ON
       ARROW_PARQUET: ON
+      ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
       ARROW_USE_GLOG: OFF
       ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
       ARROW_WITH_BROTLI: OFF
@@ -260,9 +272,6 @@ jobs:
       CMAKE_INSTALL_LIBDIR: bin
       CMAKE_INSTALL_PREFIX: /usr
       CMAKE_UNITY_BUILD: ON
-      OPENSSL_ROOT_DIR: >-
-        C:\Program Files\OpenSSL-Win64
-      NPROC: 3
     steps:
       - name: Disable Crash Dialogs
         run: |
@@ -364,6 +373,7 @@ jobs:
       CMAKE_ARGS: >-
         -DARROW_PACKAGE_PREFIX=/${{ matrix.msystem_lower}}
         -DBoost_NO_BOOST_CMAKE=ON
+        -DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON
       # We can't use unity build because we don't have enough memory on
       # GitHub Actions.
       # CMAKE_UNITY_BUILD: ON

.github/workflows/dev.yml

@@ -103,7 +103,7 @@ jobs:
         shell: bash
         run: |
           gem install test-unit
-          pip install cython setuptools six pytest jira
+          pip install "cython<3" setuptools six pytest jira
       - name: Run Release Test
         env:
           ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yml

@@ -30,17 +30,20 @@ env:
 jobs:
 
   complete:
-    name: AMD64 Ubuntu 20.04 Complete Documentation
+    name: AMD64 Ubuntu 22.04 Complete Documentation
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 150
     env:
-      UBUNTU: "20.04"
+      UBUNTU: "22.04"
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+      - name: Free up disk space
+        run: |
+          ci/scripts/util_free_space.sh
       - name: Cache Docker Volumes
         uses: actions/cache@v3
         with:

.github/workflows/issue_bot.yml

@@ -21,6 +21,7 @@ on:
   issues:
     types:
       - opened
+      - edited
 
 permissions:
   contents: read
@@ -38,6 +39,20 @@ jobs:
             let split_body = context.payload.issue.body.split('### Component(s)');
             if (split_body.length != 2) throw new Error('No components found!');
 
+            let current_labels = await github.rest.issues.listLabelsOnIssue({
+              "owner": context.repo.owner,
+              "repo": context.repo.repo,
+              "per_page": 100,
+              "issue_number": context.payload.issue.number,
+            });
+
+            let current_label_names = current_labels.data.map(label => label.name);
+
+            // keep non-component labels
+            let non_component_labels = current_label_names.filter(
+              label => !label.startsWith("Component: ")
+            );
+
             let component_labels = split_body[1]
               .split(',')
               .map(component => component.trim())
@@ -56,9 +71,9 @@ jobs:
           
             if (component_labels.length == 0) throw new Error('No components found!');
 
-            await github.rest.issues.addLabels({
+            await github.rest.issues.setLabels({
               "owner": context.repo.owner,
               "repo": context.repo.repo,
               "issue_number": context.payload.issue.number,
-              "labels": component_labels,
-            });
+              "labels": component_labels.concat(non_component_labels),
+            });

.github/workflows/java.yml

@@ -55,7 +55,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [8, 11, 17, 18]
+        jdk: [8, 11, 17, 20]
         include:
         - jdk: 8
           title: AMD64 Debian 9 Java JDK 8 Maven 3.5.4
@@ -66,13 +66,13 @@ jobs:
           maven: 3.6.2
           image: debian-java
         - jdk: 17
-          title: AMD64 Oracle Linux Server 8.5 Java JDK 17 Maven 3.8.5
-          maven: 3.8.5
-          image: oracle-java
-        - jdk: 18
-          title: AMD64 Oracle Linux Server 8.6 Java JDK 18 Maven 3.8.5
-          maven: 3.8.5
-          image: oracle-java
+          title: AMD64 Ubuntu 22.04 Java JDK 17 Maven 3.9.3
+          maven: 3.9.3
+          image: eclipse-java
+        - jdk: 20
+          title: AMD64 Ubuntu 22.04 Java JDK 20 Maven 3.9.3
+          maven: 3.9.3
+          image: eclipse-java
     env:
       JDK: ${{ matrix.jdk }}
       MAVEN: ${{ matrix.maven }}

.github/workflows/ruby.yml

@@ -229,6 +229,7 @@ jobs:
       CMAKE_ARGS: >-
         -DARROW_PACKAGE_PREFIX=/ucrt${{ matrix.mingw-n-bits }}
         -DBoost_NO_BOOST_CMAKE=ON
+        -DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON
       CMAKE_UNITY_BUILD: ON
     steps:
       - name: Disable Crash Dialogs

.pre-commit-config.yaml

@@ -37,7 +37,7 @@ repos:
         entry: --entrypoint /bin/hadolint hadolint/hadolint:latest -
         exclude: ^dev/.*$
   - repo: https://github.com/pycqa/flake8
-    rev: 5.0.3
+    rev: 6.1.0
     hooks:
       - id: flake8
         name: Python Format