From 660b888166230f40bd41143d0123471b688944b1 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Sat, 4 Feb 2023 15:44:55 +0000 Subject: [PATCH] WIP --- .github/actions/setup-builder/action.yaml | 3 + .github/workflows/arrow.yml | 220 ---------------------- .github/workflows/arrow_flight.yml | 30 +-- .github/workflows/coverage.yml | 67 ------- .github/workflows/dev.yml | 61 ------ .github/workflows/dev_pr.yml | 51 ----- .github/workflows/docs.yml | 58 ------ .github/workflows/integration.yml | 168 ----------------- .github/workflows/miri.yaml | 61 ------ .github/workflows/object_store.yml | 150 --------------- .github/workflows/parquet.yml | 171 ----------------- .github/workflows/parquet_derive.yml | 66 ------- .github/workflows/rust.yml | 102 ---------- 13 files changed, 6 insertions(+), 1202 deletions(-) delete mode 100644 .github/workflows/arrow.yml delete mode 100644 .github/workflows/coverage.yml delete mode 100644 .github/workflows/dev.yml delete mode 100644 .github/workflows/dev_pr.yml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/integration.yml delete mode 100644 .github/workflows/miri.yaml delete mode 100644 .github/workflows/object_store.yml delete mode 100644 .github/workflows/parquet.yml delete mode 100644 .github/workflows/parquet_derive.yml delete mode 100644 .github/workflows/rust.yml diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml index 865ff66b9d09..e0ec3cc5471c 100644 --- a/.github/actions/setup-builder/action.yaml +++ b/.github/actions/setup-builder/action.yaml @@ -65,3 +65,6 @@ runs: - name: Enable backtraces shell: bash run: echo "RUST_BACKTRACE=1" >> $GITHUB_ENV + - name: Fixup git permissions + shell: bash + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" diff --git a/.github/workflows/arrow.yml b/.github/workflows/arrow.yml deleted file mode 100644 index 35e70c8f070c..000000000000 --- a/.github/workflows/arrow.yml +++ /dev/null @@ -1,220 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# tests for arrow crate -name: arrow - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -on: - # always trigger - push: - branches: - - master - pull_request: - paths: - - .github/** - - arrow-arith/** - - arrow-array/** - - arrow-buffer/** - - arrow-cast/** - - arrow-csv/** - - arrow-data/** - - arrow-integration-test/** - - arrow-ipc/** - - arrow-json/** - - arrow-ord/** - - arrow-row/** - - arrow-schema/** - - arrow-select/** - - arrow-string/** - - arrow/** - -jobs: - - # test the crate - linux-test: - name: Test - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - name: Test arrow-buffer with all features - run: cargo test -p arrow-buffer --all-features - - name: Test arrow-data with all features - run: cargo test -p arrow-data --all-features - - name: Test arrow-schema with all features - run: cargo test -p arrow-schema --all-features - - name: Test arrow-array with all features except SIMD - run: cargo test -p arrow-array - - name: Test arrow-select with all features - run: cargo test -p arrow-select --all-features - - name: Test arrow-cast with all features - run: cargo test -p arrow-cast --all-features - - name: Test arrow-ipc with all features - run: cargo test -p arrow-ipc --all-features - - name: Test arrow-csv with all features - run: cargo test -p arrow-csv --all-features - - name: Test arrow-json with all features - run: cargo test -p arrow-json --all-features - - name: Test arrow-string with all features - run: cargo test -p arrow-string --all-features - - name: Test arrow-ord with all features except SIMD - run: cargo test -p arrow-ord --features dyn_cmp_dict - - name: Test arrow-arith with all features except SIMD - run: cargo test -p arrow-arith --features dyn_arith_dict - - name: Test arrow-row with all features - run: cargo test -p arrow-row --all-features - - name: Test arrow-integration-test with all features - run: cargo test -p arrow-integration-test --all-features - - name: Test arrow with default features - run: cargo test -p arrow - - name: Test arrow with all features apart from simd - run: cargo test -p arrow --features=force_validate,prettyprint,ipc_compression,ffi,dyn_cmp_dict,dyn_arith_dict,chrono-tz - - name: Run examples - run: | - # Test arrow examples - cargo run --example builders - cargo run --example dynamic_types - cargo run --example read_csv - cargo run --example read_csv_infer_schema - - name: Run non-archery based integration-tests - run: cargo test -p arrow-integration-testing - - # test compilation features - linux-features: - name: Check Compilation - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - name: Check compilation - run: cargo check -p arrow - - name: Check compilation --no-default-features - run: cargo check -p arrow --no-default-features - - name: Check compilation --all-targets - run: cargo check -p arrow --all-targets - - name: Check compilation --no-default-features --all-targets - run: cargo check -p arrow --no-default-features --all-targets - - name: Check compilation --no-default-features --all-targets --features test_utils - run: cargo check -p arrow --no-default-features --all-targets --features test_utils - - name: Check compilation --no-default-features --all-targets --features ffi - run: cargo check -p arrow --no-default-features --all-targets --features ffi - - name: Check compilation --no-default-features --all-targets --features chrono-tz - run: cargo check -p arrow --no-default-features --all-targets --features chrono-tz - - # test the --features "simd" of the arrow crate. This requires nightly Rust. - linux-test-simd: - name: Test SIMD on AMD64 Rust ${{ matrix.rust }} - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: nightly - - name: Test arrow-array with SIMD - run: cargo test -p arrow-array --features simd - - name: Test arrow-ord with SIMD - run: cargo test -p arrow-ord --features simd - - name: Test arrow-arith with SIMD - run: cargo test -p arrow-arith --features simd - - name: Test arrow with SIMD - run: cargo test -p arrow --features simd - - name: Check compilation --features simd --all-targets - run: cargo check -p arrow --features simd --all-targets - - - # test the arrow crate builds against wasm32 in nightly rust - wasm32-build: - name: Build wasm32 - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: nightly - target: wasm32-unknown-unknown,wasm32-wasi - - name: Build wasm32-unknown-unknown - run: cargo build -p arrow --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-unknown-unknown - - name: Build wasm32-wasi - run: cargo build -p arrow --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-wasi - - clippy: - name: Clippy - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - name: Setup Clippy - run: rustup component add clippy - - name: Clippy arrow-buffer with all features - run: cargo clippy -p arrow-buffer --all-targets --all-features -- -D warnings - - name: Clippy arrow-data with all features - run: cargo clippy -p arrow-data --all-targets --all-features -- -D warnings - - name: Clippy arrow-schema with all features - run: cargo clippy -p arrow-schema --all-targets --all-features -- -D warnings - - name: Clippy arrow-array with all features except SIMD - run: cargo clippy -p arrow-array --all-targets -- -D warnings - - name: Clippy arrow-select with all features - run: cargo clippy -p arrow-select --all-targets --all-features -- -D warnings - - name: Clippy arrow-cast with all features - run: cargo clippy -p arrow-cast --all-targets --all-features -- -D warnings - - name: Clippy arrow-ipc with all features - run: cargo clippy -p arrow-ipc --all-targets --all-features -- -D warnings - - name: Clippy arrow-csv with all features - run: cargo clippy -p arrow-csv --all-targets --all-features -- -D warnings - - name: Clippy arrow-json with all features - run: cargo clippy -p arrow-json --all-targets --all-features -- -D warnings - - name: Clippy arrow-string with all features - run: cargo clippy -p arrow-string --all-targets --all-features -- -D warnings - - name: Clippy arrow-ord with all features except SIMD - run: cargo clippy -p arrow-ord --all-targets --features dyn_cmp_dict -- -D warnings - - name: Clippy arrow-arith with all features except SIMD - run: cargo clippy -p arrow-arith --all-targets --features dyn_arith_dict -- -D warnings - - name: Clippy arrow-row with all features - run: cargo clippy -p arrow-row --all-targets --all-features -- -D warnings - - name: Clippy arrow with all features except SIMD - run: cargo clippy -p arrow --features=prettyprint,csv,ipc,test_utils,ffi,ipc_compression,dyn_cmp_dict,dyn_arith_dict,chrono-tz --all-targets -- -D warnings - - name: Clippy arrow-integration-test with all features - run: cargo clippy -p arrow-integration-test --all-targets --all-features -- -D warnings - - name: Clippy arrow-integration-testing with all features - run: cargo clippy -p arrow-integration-testing --all-targets --all-features -- -D warnings diff --git a/.github/workflows/arrow_flight.yml b/.github/workflows/arrow_flight.yml index 02c149aaae0b..c1f86e240286 100644 --- a/.github/workflows/arrow_flight.yml +++ b/.github/workflows/arrow_flight.yml @@ -54,30 +54,6 @@ jobs: - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - name: Test - run: | - cargo test -p arrow-flight - - name: Test --all-features - run: | - cargo test -p arrow-flight --all-features - - name: Test --examples - run: | - cargo test -p arrow-flight --features=flight-sql-experimental --examples - - name: Test --examples with TLS - run: | - cargo test -p arrow-flight --features=flight-sql-experimental,tls --examples - - name: Verify workspace clean - run: git diff --exit-code - - clippy: - name: Clippy - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - name: Setup Clippy - run: rustup component add clippy - - name: Run clippy - run: cargo clippy -p arrow-flight --all-targets --all-features -- -D warnings + run : ls -all + - name: Test2 + run : git status diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml deleted file mode 100644 index 3fa254142dbe..000000000000 --- a/.github/workflows/coverage.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: coverage - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -# Trigger only on pushes to master, not pull requests -on: - push: - branches: - - master - -jobs: - - coverage: - name: Coverage - runs-on: ubuntu-latest - # Note runs outside of a container - # otherwise we get this error: - # Failed to run tests: ASLR disable failed: EPERM: Operation not permitted - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - run: | - rustup toolchain install stable - rustup default stable - - name: Install protobuf compiler in /protoc - run: | - sudo mkdir /protoc - sudo chmod a+rwx /protoc - cd /protoc - curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protoc-21.4-linux-x86_64.zip - unzip protoc-21.4-linux-x86_64.zip - - name: Cache Cargo - uses: actions/cache@v3 - with: - path: /home/runner/.cargo - key: cargo-coverage-cache3- - - name: Run coverage - run: | - export PATH=$PATH:/protoc/bin - rustup toolchain install stable - rustup default stable - cargo install --version 0.18.2 cargo-tarpaulin - cargo tarpaulin --all --out Xml - - name: Report coverage - continue-on-error: true - run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml deleted file mode 100644 index 0eb2d024f352..000000000000 --- a/.github/workflows/dev.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: dev - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -# trigger for all PRs and changes to master -on: - push: - branches: - - master - pull_request: - -env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - rat: - name: Release Audit Tool (RAT) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Audit licenses - run: ./dev/release/run-rat.sh . - - prettier: - name: Markdown format - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: "14" - - name: Prettier check - run: | - # if you encounter error, run the command below and commit the changes - npx prettier@2.3.2 --write {arrow,arrow-flight,dev,arrow-integration-testing,parquet}/**/*.md README.md CODE_OF_CONDUCT.md CONTRIBUTING.md - git diff --exit-code diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml deleted file mode 100644 index c1492580cd39..000000000000 --- a/.github/workflows/dev_pr.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: dev_pr - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -# Trigger whenever a PR is changed (title as well as new / changed commits) -on: - pull_request_target: - types: - - opened - - edited - - synchronize - -jobs: - process: - name: Process - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - steps: - - uses: actions/checkout@v3 - - - name: Assign GitHub labels - if: | - github.event_name == 'pull_request_target' && - (github.event.action == 'opened' || - github.event.action == 'synchronize') - uses: actions/labeler@4.1.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - configuration-path: .github/workflows/dev_pr/labeler.yml - sync-labels: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index bf1bf7aad880..000000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: docs - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -# trigger for all PRs and changes to master -on: - push: - branches: - - master - pull_request: - -jobs: - - # test doc links still work - docs: - name: Rustdocs are clean - runs-on: ubuntu-latest - strategy: - matrix: - arch: [ amd64 ] - rust: [ nightly ] - container: - image: ${{ matrix.arch }}/rust - env: - RUSTDOCFLAGS: "-Dwarnings" - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Install python dev - run: | - apt update - apt install -y libpython3.9-dev - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{ matrix.rust }} - - name: Run cargo doc - run: cargo doc --document-private-items --no-deps --workspace --all-features diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index 9b2e7797d5ff..000000000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,168 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: integration - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -# trigger for all PRs that touch certain files and changes to master -on: - push: - branches: - - master - pull_request: - paths: - - .github/** - - arrow-array/** - - arrow-buffer/** - - arrow-cast/** - - arrow-csv/** - - arrow-data/** - - arrow-integration-test/** - - arrow-integration-testing/** - - arrow-ipc/** - - arrow-json/** - - arrow-ord/** - - arrow-pyarrow-integration-testing/** - - arrow-schema/** - - arrow-select/** - - arrow-sort/** - - arrow-string/** - - arrow/** - -jobs: - - integration: - name: Archery test With other arrows - runs-on: ubuntu-latest - container: - image: apache/arrow-dev:amd64-conda-integration - env: - ARROW_USE_CCACHE: OFF - ARROW_CPP_EXE_PATH: /build/cpp/debug - BUILD_DOCS_CPP: OFF - # These are necessary because the github runner overrides $HOME - # https://github.com/actions/runner/issues/863 - RUSTUP_HOME: /root/.rustup - CARGO_HOME: /root/.cargo - defaults: - run: - shell: bash - steps: - # This is necessary so that actions/checkout can find git - - name: Export conda path - run: echo "/opt/conda/envs/arrow/bin" >> $GITHUB_PATH - # This is necessary so that Rust can find cargo - - name: Export cargo path - run: echo "/root/.cargo/bin" >> $GITHUB_PATH - - name: Check rustup - run: which rustup - - name: Check cmake - run: which cmake - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - repository: apache/arrow - submodules: true - fetch-depth: 0 - - name: Checkout Arrow Rust - uses: actions/checkout@v3 - with: - path: rust - fetch-depth: 0 - - name: Make build directory - run: mkdir /build - - name: Build Rust - run: conda run --no-capture-output ci/scripts/rust_build.sh $PWD /build - - name: Build C++ - run: conda run --no-capture-output ci/scripts/cpp_build.sh $PWD /build - - name: Build C# - run: conda run --no-capture-output ci/scripts/csharp_build.sh $PWD /build - - name: Build Go - run: conda run --no-capture-output ci/scripts/go_build.sh $PWD - - name: Build Java - run: conda run --no-capture-output ci/scripts/java_build.sh $PWD /build - - name: Build JS - run: conda run --no-capture-output ci/scripts/js_build.sh $PWD /build - - name: Install archery - run: conda run --no-capture-output pip install -e dev/archery - - name: Run integration tests - run: | - conda run --no-capture-output archery integration \ - --run-flight \ - --with-cpp=1 \ - --with-csharp=1 \ - --with-java=1 \ - --with-js=1 \ - --with-go=1 \ - --with-rust=1 \ - --gold-dirs=testing/data/arrow-ipc-stream/integration/0.14.1 \ - --gold-dirs=testing/data/arrow-ipc-stream/integration/0.17.1 \ - --gold-dirs=testing/data/arrow-ipc-stream/integration/1.0.0-bigendian \ - --gold-dirs=testing/data/arrow-ipc-stream/integration/1.0.0-littleendian \ - --gold-dirs=testing/data/arrow-ipc-stream/integration/2.0.0-compression \ - --gold-dirs=testing/data/arrow-ipc-stream/integration/4.0.0-shareddict - - # test FFI against the C-Data interface exposed by pyarrow - pyarrow-integration-test: - name: Pyarrow C Data Interface - runs-on: ubuntu-latest - strategy: - matrix: - rust: [ stable ] - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - run: | - rustup toolchain install ${{ matrix.rust }} - rustup default ${{ matrix.rust }} - rustup component add rustfmt clippy - - name: Cache Cargo - uses: actions/cache@v3 - with: - path: /home/runner/.cargo - key: cargo-maturin-cache- - - name: Cache Rust dependencies - uses: actions/cache@v3 - with: - path: /home/runner/target - # this key is not equal because maturin uses different compilation flags. - key: ${{ runner.os }}-${{ matrix.arch }}-target-maturin-cache-${{ matrix.rust }}- - - uses: actions/setup-python@v4 - with: - python-version: '3.7' - - name: Upgrade pip and setuptools - run: pip install --upgrade pip setuptools wheel virtualenv - - name: Create virtualenv and install dependencies - run: | - virtualenv venv - source venv/bin/activate - pip install maturin toml pytest pytz pyarrow>=5.0 - - name: Run Rust tests - run: | - source venv/bin/activate - cargo test -p arrow --test pyarrow --features pyarrow - - name: Run tests - run: | - source venv/bin/activate - cd arrow-pyarrow-integration-testing - maturin develop - pytest -v . diff --git a/.github/workflows/miri.yaml b/.github/workflows/miri.yaml deleted file mode 100644 index 0c1f8069cd40..000000000000 --- a/.github/workflows/miri.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: miri - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -# trigger for all PRs that touch certain files and changes to master -on: - push: - branches: - - master - pull_request: - paths: - - .github/** - - arrow-array/** - - arrow-buffer/** - - arrow-cast/** - - arrow-csv/** - - arrow-data/** - - arrow-ipc/** - - arrow-json/** - - arrow-schema/** - - arrow-select/** - - arrow-string/** - - arrow/** - -jobs: - miri-checks: - name: MIRI - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - run: | - rustup toolchain install nightly --component miri - rustup override set nightly - cargo miri setup - - name: Run Miri Checks - env: - RUST_BACKTRACE: full - RUST_LOG: "trace" - run: bash .github/workflows/miri.sh diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml deleted file mode 100644 index f182d21eef13..000000000000 --- a/.github/workflows/object_store.yml +++ /dev/null @@ -1,150 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ---- -# tests for `object_store` crate -name: object_store - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -# trigger for all PRs that touch certain files and changes to master -on: - push: - branches: - - master - pull_request: - paths: - - object_store/** - - .github/** - -jobs: - clippy: - name: Clippy - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - name: Setup Clippy - run: rustup component add clippy - # Run different tests for the library on its own as well as - # all targets to ensure that it still works in the absence of - # features that might be enabled by dev-dependencies of other - # targets. - - name: Run clippy with default features - run: cargo clippy -p object_store -- -D warnings - - name: Run clippy with aws feature - run: cargo clippy -p object_store --features aws -- -D warnings - - name: Run clippy with aws_profile feature - run: cargo clippy -p object_store --features aws_profile -- -D warnings - - name: Run clippy with gcp feature - run: cargo clippy -p object_store --features gcp -- -D warnings - - name: Run clippy with azure feature - run: cargo clippy -p object_store --features azure -- -D warnings - - name: Run clippy with all features - run: cargo clippy -p object_store --all-features -- -D warnings - - name: Run clippy with all features and all targets - run: cargo clippy -p object_store --all-features --all-targets -- -D warnings - - # test the crate - # This runs outside a container to workaround lack of support for passing arguments - # to service containers - https://github.com/orgs/community/discussions/26688 - linux-test: - name: Emulator Tests - runs-on: ubuntu-latest - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - RUST_BACKTRACE: "1" - # Run integration tests - TEST_INTEGRATION: 1 - EC2_METADATA_ENDPOINT: http://localhost:1338 - AZURE_USE_EMULATOR: "1" - AZURITE_BLOB_STORAGE_URL: "http://localhost:10000" - AZURITE_QUEUE_STORAGE_URL: "http://localhost:10001" - HTTP_URL: "http://localhost:8080" - GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json" - OBJECT_STORE_BUCKET: test-bucket - - steps: - - uses: actions/checkout@v3 - - - name: Configure Fake GCS Server (GCP emulation) - run: | - docker run -d -p 4443:4443 fsouza/fake-gcs-server -scheme http - # Give the container a moment to start up prior to configuring it - sleep 1 - curl -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "http://localhost:4443/storage/v1/b" - echo '{"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT" - - - name: Setup WebDav - run: docker run -d -p 8080:80 rclone/rclone serve webdav /data --addr :80 - - - name: Setup LocalStack (AWS emulation) - env: - AWS_DEFAULT_REGION: "us-east-1" - AWS_ACCESS_KEY_ID: test - AWS_SECRET_ACCESS_KEY: test - AWS_ENDPOINT: http://localhost:4566 - run: | - docker run -d -p 4566:4566 localstack/localstack:0.14.4 - docker run -d -p 1338:1338 amazon/amazon-ec2-metadata-mock:v1.9.2 --imdsv2 - aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket - - - name: Configure Azurite (Azure emulation) - # the magical connection string is from - # https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings - run: | - docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite - az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;' - - - name: Setup Rust toolchain - run: | - rustup toolchain install stable - rustup default stable - - - name: Run object_store tests - env: - OBJECT_STORE_AWS_DEFAULT_REGION: "us-east-1" - OBJECT_STORE_AWS_ACCESS_KEY_ID: test - OBJECT_STORE_AWS_SECRET_ACCESS_KEY: test - OBJECT_STORE_AWS_ENDPOINT: http://localhost:4566 - run: cargo test -p object_store --features=aws,azure,gcp,http - - # test the object_store crate builds against wasm32 in stable rust - wasm32-build: - name: Build wasm32 - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - target: wasm32-unknown-unknown,wasm32-wasi - - name: Build wasm32-unknown-unknown - run: cargo build -p object_store --target wasm32-unknown-unknown - - name: Build wasm32-wasi - run: cargo build -p object_store --target wasm32-wasi \ No newline at end of file diff --git a/.github/workflows/parquet.yml b/.github/workflows/parquet.yml deleted file mode 100644 index ee5813f567bb..000000000000 --- a/.github/workflows/parquet.yml +++ /dev/null @@ -1,171 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ---- -# tests for parquet crate -name: "parquet" - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -# trigger for all PRs that touch certain files and changes to master -on: - push: - branches: - - master - pull_request: - paths: - - arrow/** - - arrow-array/** - - arrow-buffer/** - - arrow-cast/** - - arrow-data/** - - arrow-schema/** - - arrow-select/** - - arrow-ipc/** - - arrow-csv/** - - arrow-json/** - - parquet/** - - .github/** - -jobs: - # test the crate - linux-test: - name: Test - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - name: Test - run: cargo test -p parquet - - name: Test --all-features - run: cargo test -p parquet --all-features - - # test compilation - linux-features: - name: Check Compilation - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - # Run different tests for the library on its own as well as - # all targets to ensure that it still works in the absence of - # features that might be enabled by dev-dependencies of other - # targets. - # - # This for each of (library and all-targets), check - # 1. compiles with default features - # 1. compiles with no default features - # 3. compiles with just arrow feature - # 3. compiles with all features - - name: Check compilation - run: cargo check -p parquet - - name: Check compilation --no-default-features - run: cargo check -p parquet --no-default-features - - name: Check compilation --no-default-features --features arrow - run: cargo check -p parquet --no-default-features --features arrow - - name: Check compilation --no-default-features --all-features - run: cargo check -p parquet --all-features - - name: Check compilation --all-targets - run: cargo check -p parquet --all-targets - - name: Check compilation --all-targets --no-default-features - run: cargo check -p parquet --all-targets --no-default-features - - name: Check compilation --all-targets --no-default-features --features arrow - run: cargo check -p parquet --all-targets --no-default-features --features arrow - - name: Check compilation --all-targets --all-features - run: cargo check -p parquet --all-targets --all-features - - name: Check compilation --all-targets --no-default-features --features json - run: cargo check -p parquet --all-targets --no-default-features --features json - - # test the parquet crate builds against wasm32 in stable rust - wasm32-build: - name: Build wasm32 - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - target: wasm32-unknown-unknown,wasm32-wasi - - name: Build wasm32-unknown-unknown - run: cargo build -p parquet --no-default-features --features cli,snap,flate2,brotli --target wasm32-unknown-unknown - - name: Build wasm32-wasi - run: cargo build -p parquet --no-default-features --features cli,snap,flate2,brotli --target wasm32-wasi - - pyspark-integration-test: - name: PySpark Integration Test - runs-on: ubuntu-latest - strategy: - matrix: - rust: [ stable ] - steps: - - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - name: Install Python dependencies - run: | - cd parquet/pytest - pip install -r requirements.txt - - name: Black check the test files - run: | - cd parquet/pytest - black --check *.py --verbose - - name: Setup Rust toolchain - run: | - rustup toolchain install ${{ matrix.rust }} - rustup default ${{ matrix.rust }} - - name: Install binary for checking - run: | - cargo install --path parquet --bin parquet-show-bloom-filter --features=cli - cargo install --path parquet --bin parquet-fromcsv --features=arrow,cli - - name: Run pytest - run: | - cd parquet/pytest - pytest -v - - clippy: - name: Clippy - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - name: Setup Clippy - run: rustup component add clippy - - name: Run clippy - run: cargo clippy -p parquet --all-targets --all-features -- -D warnings diff --git a/.github/workflows/parquet_derive.yml b/.github/workflows/parquet_derive.yml deleted file mode 100644 index 72b90ecfd81a..000000000000 --- a/.github/workflows/parquet_derive.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ---- -# tests for parquet_derive crate -name: parquet_derive - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -# trigger for all PRs that touch certain files and changes to master -on: - push: - branches: - - master - pull_request: - paths: - - parquet/** - - parquet_derive/** - - parquet_derive_test/** - - .github/** - -jobs: - # test the crate - linux-test: - name: Test - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - name: Test - run: cargo test -p parquet_derive - - clippy: - name: Clippy - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - name: Setup Clippy - run: rustup component add clippy - - name: Run clippy - run: cargo clippy -p parquet_derive --all-features -- -D warnings diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index e09e898fe160..000000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,102 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# workspace wide tests -name: rust - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -# trigger for all PRs and changes to master -on: - push: - branches: - - master - pull_request: - -jobs: - - # Check workspace wide compile and test with default features for - # mac - macos: - name: Test on Mac - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Install protoc with brew - run: brew install protobuf - - name: Setup Rust toolchain - run: | - rustup toolchain install stable --no-self-update - rustup default stable - - name: Run tests - shell: bash - run: | - # do not produce debug symbols to keep memory usage down - export RUSTFLAGS="-C debuginfo=0" - cargo test - - - # Check workspace wide compile and test with default features for - # windows - windows: - name: Test on Windows - runs-on: windows-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Install protobuf compiler in /d/protoc - shell: bash - run: | - mkdir /d/protoc - cd /d/protoc - curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protoc-21.4-win64.zip - unzip protoc-21.4-win64.zip - export PATH=$PATH:/d/protoc/bin - protoc --version - - - name: Setup Rust toolchain - run: | - rustup toolchain install stable --no-self-update - rustup default stable - - name: Run tests - shell: bash - run: | - # do not produce debug symbols to keep memory usage down - export RUSTFLAGS="-C debuginfo=0" - export PATH=$PATH:/d/protoc/bin - cargo test - - - # Run cargo fmt for all crates - lint: - name: Lint (cargo fmt) - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - name: Setup rustfmt - run: rustup component add rustfmt - - name: Run - run: cargo fmt --all -- --check