Merge branch 'apache:master' into master

.asf.yaml

@@ -38,4 +38,10 @@ github:
         # require branches to be up-to-date before merging
         strict: true
         # don't require any jobs to pass
-        contexts: []
+        contexts: []
+
+# publishes the content of the `asf-site` branch to
+# https://arrow.apache.org/rust/
+publish:
+  whoami: asf-site
+  subdir: rust

.gitattributes

@@ -1,6 +1,3 @@
-r/R/RcppExports.R linguist-generated=true
-r/R/arrowExports.R linguist-generated=true
-r/src/RcppExports.cpp linguist-generated=true
-r/src/arrowExports.cpp linguist-generated=true
-r/man/*.Rd linguist-generated=true
-
+parquet/src/format.rs linguist-generated
+arrow-flight/src/arrow.flight.protocol.rs linguist-generated
+arrow-flight/src/sql/arrow.flight.protocol.sql.rs linguist-generated

.github/actions/setup-builder/action.yaml

@@ -20,8 +20,12 @@ description: 'Prepare Rust Build Environment'
 inputs:
   rust-version:
     description: 'version of rust to install (e.g. stable)'
-    required: true
+    required: false
     default: 'stable'
+  target:
+    description: 'target architecture(s)'
+    required: false
+    default: 'x86_64-unknown-linux-gnu'
 runs:
   using: "composite"
   steps:
@@ -51,6 +55,17 @@ runs:
       shell: bash
       run: |
         echo "Installing ${{ inputs.rust-version }}"
-        rustup toolchain install ${{ inputs.rust-version }}
+        rustup toolchain install ${{ inputs.rust-version }} --target ${{ inputs.target }}
         rustup default ${{ inputs.rust-version }}
-        echo "CARGO_TARGET_DIR=/github/home/target" >> $GITHUB_ENV
+    - name: Disable debuginfo generation
+      # Disable full debug symbol generation to speed up CI build and keep memory down
+      # "1" means line tables only, which is useful for panic tracebacks.
+      shell: bash
+      run: echo "RUSTFLAGS=-C debuginfo=1" >> $GITHUB_ENV
+    - name: Enable backtraces
+      shell: bash
+      run: echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
+    - name: Fixup git permissions
+      # https://github.com/actions/checkout/issues/766
+      shell: bash
+      run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

.github/dependabot.yml

@@ -6,10 +6,17 @@ updates:
       interval: daily
     open-pull-requests-limit: 10
     target-branch: master
-    labels: [auto-dependencies]
+    labels: [ auto-dependencies, arrow ]
+  - package-ecosystem: cargo
+    directory: "/object_store"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    target-branch: master
+    labels: [ auto-dependencies, object_store ]
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 10
-    labels: [auto-dependencies]
+    labels: [ auto-dependencies ]

.github/workflows/arrow.yml

@@ -18,15 +18,33 @@
 # tests for arrow crate
 name: arrow
 
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 on:
   # always trigger
   push:
     branches:
       - master
   pull_request:
     paths:
-      - arrow/**
       - .github/**
+      - arrow-arith/**
+      - arrow-array/**
+      - arrow-buffer/**
+      - arrow-cast/**
+      - arrow-csv/**
+      - arrow-data/**
+      - arrow-integration-test/**
+      - arrow-ipc/**
+      - arrow-json/**
+      - arrow-ord/**
+      - arrow-row/**
+      - arrow-schema/**
+      - arrow-select/**
+      - arrow-string/**
+      - arrow/**
 
 jobs:
 
@@ -36,140 +54,165 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
-      - name: Test
-        run: |
-          cargo test -p arrow
-      - name: Test --features=force_validate,prettyprint,ffi
-        run: |
-          cargo test -p arrow --features=force_validate,prettyprint,ffi
+      - name: Test arrow-buffer with all features
+        run: cargo test -p arrow-buffer --all-features
+      - name: Test arrow-data with all features
+        run: cargo test -p arrow-data --all-features
+      - name: Test arrow-schema with all features
+        run: cargo test -p arrow-schema --all-features
+      - name: Test arrow-array with all features except SIMD
+        run: cargo test -p arrow-array
+      - name: Test arrow-select with all features
+        run: cargo test -p arrow-select --all-features
+      - name: Test arrow-cast with all features
+        run: cargo test -p arrow-cast --all-features
+      - name: Test arrow-ipc with all features
+        run: cargo test -p arrow-ipc --all-features
+      - name: Test arrow-csv with all features
+        run: cargo test -p arrow-csv --all-features
+      - name: Test arrow-json with all features
+        run: cargo test -p arrow-json --all-features
+      - name: Test arrow-string with all features
+        run: cargo test -p arrow-string --all-features
+      - name: Test arrow-ord with all features
+        run: cargo test -p arrow-ord --all-features
+      - name: Test arrow-arith with all features except SIMD
+        run: cargo test -p arrow-arith
+      - name: Test arrow-row with all features
+        run: cargo test -p arrow-row --all-features
+      - name: Test arrow-integration-test with all features
+        run: cargo test -p arrow-integration-test --all-features
+      - name: Test arrow with default features
+        run: cargo test -p arrow
+      - name: Test arrow with all features apart from simd
+        run: cargo test -p arrow --features=force_validate,prettyprint,ipc_compression,ffi,chrono-tz
       - name: Run examples
         run: |
           # Test arrow examples
           cargo run --example builders
           cargo run --example dynamic_types
           cargo run --example read_csv
           cargo run --example read_csv_infer_schema
+      - name: Run non-archery based integration-tests
+        run: cargo test -p arrow-integration-testing
 
-  # test compilaton features
+  # test compilation features
   linux-features:
     name: Check Compilation
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
       - name: Check compilation
-        run: |
-          cargo check -p arrow
+        run: cargo check -p arrow
       - name: Check compilation --no-default-features
-        run: |
-          cargo check -p arrow --no-default-features
+        run: cargo check -p arrow --no-default-features
       - name: Check compilation --all-targets
-        run: |
-          cargo check -p arrow --all-targets
+        run: cargo check -p arrow --all-targets
       - name: Check compilation --no-default-features --all-targets
-        run: |
-          cargo check -p arrow --no-default-features --all-targets
+        run: cargo check -p arrow --no-default-features --all-targets
       - name: Check compilation --no-default-features --all-targets --features test_utils
-        run: |
-          cargo check -p arrow --no-default-features --all-targets --features test_utils
+        run: cargo check -p arrow --no-default-features --all-targets --features test_utils
+      - name: Check compilation --no-default-features --all-targets --features ffi
+        run: cargo check -p arrow --no-default-features --all-targets --features ffi
+      - name: Check compilation --no-default-features --all-targets --features chrono-tz
+        run: cargo check -p arrow --no-default-features --all-targets --features chrono-tz
 
   # test the --features "simd" of the arrow crate. This requires nightly Rust.
   linux-test-simd:
     name: Test SIMD on AMD64 Rust ${{ matrix.rust }}
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
           rust-version: nightly
-      - name: Run tests --features "simd"
-        run: |
-          cargo test -p arrow --features "simd"
-      - name: Check compilation --features "simd"
-        run: |
-          cargo check -p arrow --features simd
+      - name: Test arrow-array with SIMD
+        run: cargo test -p arrow-array --features simd
+      - name: Test arrow-arith with SIMD
+        run: cargo test -p arrow-arith --features simd
+      - name: Test arrow with SIMD
+        run: cargo test -p arrow --features simd
       - name: Check compilation --features simd --all-targets
-        run: |
-          cargo check -p arrow --features simd --all-targets
+        run: cargo check -p arrow --features simd --all-targets
 
 
-  # test the arrow crate builds against wasm32 in stable rust
+  # test the arrow crate builds against wasm32 in nightly rust
   wasm32-build:
     name: Build wasm32
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
-      - name: Cache Cargo
-        uses: actions/cache@v3
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
         with:
-          path: /github/home/.cargo
-          key: cargo-wasm32-cache3-
-      - name: Setup Rust toolchain for WASM
-        run: |
-          rustup toolchain install nightly
-          rustup override set nightly
-          rustup target add wasm32-unknown-unknown
-          rustup target add wasm32-wasi
-      - name: Build
-        run: |
-          cd arrow
-          cargo build --no-default-features --features=csv,ipc,simd,ffi --target wasm32-unknown-unknown
-          cargo build --no-default-features --features=csv,ipc,simd,ffi --target wasm32-wasi
+          rust-version: nightly
+          target: wasm32-unknown-unknown,wasm32-wasi
+      - name: Build wasm32-unknown-unknown
+        run: cargo build -p arrow --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-unknown-unknown
+      - name: Build wasm32-wasi
+        run: cargo build -p arrow --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-wasi
 
   clippy:
     name: Clippy
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
       - name: Setup Clippy
-        run: |
-          rustup component add clippy
-      - name: Run clippy
-        run: |
-          cargo clippy -p arrow --features=prettyprint,csv,ipc,test_utils,ffi --all-targets -- -D warnings
+        run: rustup component add clippy
+      - name: Clippy arrow-buffer with all features
+        run: cargo clippy -p arrow-buffer --all-targets --all-features -- -D warnings
+      - name: Clippy arrow-data with all features
+        run: cargo clippy -p arrow-data --all-targets --all-features -- -D warnings
+      - name: Clippy arrow-schema with all features
+        run: cargo clippy -p arrow-schema --all-targets --all-features -- -D warnings
+      - name: Clippy arrow-array with all features except SIMD
+        run: cargo clippy -p arrow-array --all-targets -- -D warnings
+      - name: Clippy arrow-select with all features
+        run: cargo clippy -p arrow-select --all-targets --all-features -- -D warnings
+      - name: Clippy arrow-cast with all features
+        run: cargo clippy -p arrow-cast --all-targets --all-features -- -D warnings
+      - name: Clippy arrow-ipc with all features
+        run: cargo clippy -p arrow-ipc --all-targets --all-features -- -D warnings
+      - name: Clippy arrow-csv with all features
+        run: cargo clippy -p arrow-csv --all-targets --all-features -- -D warnings
+      - name: Clippy arrow-json with all features
+        run: cargo clippy -p arrow-json --all-targets --all-features -- -D warnings
+      - name: Clippy arrow-string with all features
+        run: cargo clippy -p arrow-string --all-targets --all-features -- -D warnings
+      - name: Clippy arrow-ord with all features
+        run: cargo clippy -p arrow-ord --all-targets --all-features -- -D warnings
+      - name: Clippy arrow-arith with all features except SIMD
+        run: cargo clippy -p arrow-arith --all-targets -- -D warnings
+      - name: Clippy arrow-row with all features
+        run: cargo clippy -p arrow-row --all-targets --all-features -- -D warnings
+      - name: Clippy arrow with all features except SIMD
+        run: cargo clippy -p arrow --features=prettyprint,csv,ipc,test_utils,ffi,ipc_compression,chrono-tz --all-targets -- -D warnings
+      - name: Clippy arrow-integration-test with all features
+        run: cargo clippy -p arrow-integration-test --all-targets --all-features -- -D warnings
+      - name: Clippy arrow-integration-testing with all features
+        run: cargo clippy -p arrow-integration-testing --all-targets --all-features -- -D warnings