Merge remote-tracking branch 'origin/master' into flight_data_compres…

…sion

.github/actions/setup-builder/action.yaml

@@ -25,6 +25,31 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - name: Cache Cargo
+      uses: actions/cache@v3
+      with:
+        # these represent dependencies downloaded by cargo
+        # and thus do not depend on the OS, arch nor rust version.
+        #
+        # source https://github.com/actions/cache/blob/main/examples.md#rust---cargo
+        path: |
+          /usr/local/cargo/bin/
+          /usr/local/cargo/registry/index/
+          /usr/local/cargo/registry/cache/
+          /usr/local/cargo/git/db/
+        key: cargo-cache3-${{ hashFiles('**/Cargo.toml') }}
+        restore-keys: cargo-cache3-
+    - name: Generate lockfile
+      shell: bash
+      run: cargo fetch
+    - name: Cache Rust dependencies
+      uses: actions/cache@v3
+      with:
+        # these represent compiled steps of both dependencies and arrow
+        # and thus are specific for a particular OS, arch and rust version.
+        path: /github/home/target
+        key: ${{ runner.os }}-${{ runner.arch }}-target-cache3-${{ inputs.rust-version }}-${{ hashFiles('**/Cargo.lock') }}
+        restore-keys: ${{ runner.os }}-${{ runner.arch }}-target-cache3-${{ inputs.rust-version }}-
     - name: Install Build Dependencies
       shell: bash
       run: |
@@ -36,4 +61,4 @@ runs:
         echo "Installing ${{ inputs.rust-version }}"
         rustup toolchain install ${{ inputs.rust-version }}
         rustup default ${{ inputs.rust-version }}
-        rustup component add rustfmt
+        echo "CARGO_TARGET_DIR=/github/home/target" >> $GITHUB_ENV

.github/workflows/integration.yml

@@ -39,7 +39,7 @@ jobs:
           path: rust
           fetch-depth: 0
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: 3.8
       - name: Setup Archery
@@ -64,17 +64,17 @@ jobs:
           rustup default ${{ matrix.rust }}
           rustup component add rustfmt clippy
       - name: Cache Cargo
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /home/runner/.cargo
           key: cargo-maturin-cache-
       - name: Cache Rust dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /home/runner/target
           # this key is not equal because maturin uses different compilation flags.
           key: ${{ runner.os }}-${{ matrix.arch }}-target-maturin-cache-${{ matrix.rust }}-
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v3
         with:
           python-version: '3.7'
       - name: Upgrade pip and setuptools

.github/workflows/miri.sh

@@ -6,21 +6,12 @@
 # rustup default nightly
 
 
-export MIRIFLAGS="-Zmiri-disable-isolation"
+# stacked borrows checking uses too much memory to run successfully in github actions
+# re-enable if the CI is migrated to something more powerful (https://github.com/apache/arrow-rs/issues/1833)
+# see also https://github.com/rust-lang/miri/issues/1367
+export MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-stacked-borrows"
 cargo miri setup
 cargo clean
 
-run_miri() {
-    # Currently only the arrow crate is tested with miri
-    # IO related tests and some unsupported tests are skipped
-    cargo miri test -p arrow -- --skip csv --skip ipc --skip json
-}
-
-# If MIRI fails, automatically retry
-# Seems like miri is occasionally killed by the github runner
-# https://github.com/apache/arrow-rs/issues/879
-for i in `seq 1 5`; do
-    echo "Starting Arrow MIRI run..."
-    run_miri && break
-    echo "foo" > /tmp/data.txt
-done
+echo "Starting Arrow MIRI run..."
+cargo miri test -p arrow -- --skip csv --skip ipc --skip json

.github/workflows/miri.yaml

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: Rust
+name: MIRI
 
 on:
   # always trigger
@@ -26,19 +26,15 @@ jobs:
   miri-checks:
     name: MIRI
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        arch: [amd64]
-        rust: [nightly-2022-01-17]
     steps:
       - uses: actions/checkout@v2
         with:
           submodules: true
       - name: Setup Rust toolchain
         run: |
-          rustup toolchain install ${{ matrix.rust }}
-          rustup default ${{ matrix.rust }}
-          rustup component add rustfmt clippy miri
+          rustup toolchain install nightly --component miri
+          rustup override set nightly
+          cargo miri setup
       - name: Run Miri Checks
         env:
           RUST_BACKTRACE: full