test: add integration test with minio

.github/workflows/ci.yml

@@ -70,6 +70,19 @@ jobs:
           name: cov-report-rust-tests-${{ runner.os }}
           path: ./cov-reports
           if-no-files-found: 'error'
+
+  rust-integration-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-22.04 ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Spin up minio
+        run: docker compose -f docker/docker-compose.yaml up -d
+      - name: Run Rust integration tests
+        run: cd integration_test && cargo test --config env.toml
 
   python-tests:
     strategy:

.gitignore

@@ -16,6 +16,7 @@
 # under the License.
 
 /Cargo.lock
+**/Cargo.lock
 /target
 **/target
 

crates/tests/src/lib.rs

@@ -61,6 +61,13 @@
         path_buf.to_str().unwrap().to_string()
     }
 
+    pub fn s3_path(&self) -> String {
+        let bucket = std::env::var("INTEGRATION_TEST_S3_BUCKET")
+            .expect("INTEGRATION_TEST_S3_BUCKET not set");
+        let data_path = Path::new(format!("s3://{}", bucket).as_str()).join(self.as_ref());
+        data_path.to_str().unwrap().to_string()
+    }
+
     pub fn url(&self) -> Url {
         let path = self.path();
         Url::from_file_path(path).unwrap()

docker/Dockerfile

@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM alpine:3.18
+RUN apk update && apk add --no-cache wget ca-certificates bash findutils
+
+RUN cd /usr/local/bin && \
+    wget -q --show-progress https://dl.min.io/client/mc/release/linux-amd64/mc && \
+    chmod +x mc

docker/copy_tables.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eux
+
+echo ">>> Creating Minio store"
+mc config host add store http://minio:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY
+
+echo ">>> Creating bucket"
+mc mb store/$MINIO_TEST_BUCKET
+
+echo ">>> Unzipping tables"
+find /opt/data -name "*.zip" -exec unzip {} -d /tmp/data/ \;
+
+echo ">>> Copying tables to Minio"
+find /tmp/data -type d -mindepth 1 -maxdepth 1 -exec mc cp --recursive {} store/$MINIO_TEST_BUCKET \;
+
+echo ">>> Listing uploaded tables"
+mc ls store/$MINIO_TEST_BUCKET

docker/docker-compose.yaml

@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: "3.8"
+
+services:
+  minio:
+    image: quay.io/minio/minio
+    container_name: minio
+    ports:
+      - "9000:9000"
+    env_file:
+      - s3.env
+    command: server /data
+    healthcheck:
+      test: mc ready local
+      interval: 5s
+      timeout: 10s
+      retries: 5
+      start_period: 2s
+
+
+  mc:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: mc
+    depends_on:
+      minio:
+        condition: service_healthy
+    env_file:
+      - s3.env
+    volumes:
+      - ../crates/tests/data/tables:/opt/data:ro
+      - ./copy_tables.sh:/opt/copy_tables.sh
+    entrypoint: /opt/copy_tables.sh
+

docker/s3.env

@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+MINIO_ROOT_USER=minio
+MINIO_ROOT_PASSWORD=minio123
+MINIO_ACCESS_KEY=minio
+MINIO_SECRET_KEY=minio123
+MINIO_TEST_BUCKET=test-bucket

integration_test/Cargo.toml

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[workspace]
+members = []
+
+[package]
+version = "0.1.0"
+name = "integration_test"
+edition = "2021"
+license = "Apache-2.0"
+rust-version = "1.75.0"
+description = "Integration tests for hudi-rs"
+
+
+[dependencies]
+hudi = { features=["datafusion"], path="../crates/hudi" }
+hudi-tests = { path="../crates/tests" }
+datafusion = { version = "= 39.0.0" }
+datafusion-expr = { version = "= 39.0.0" }
+tokio = { version = "1", features = ["rt-multi-thread"] }

integration_test/env.toml

@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[env]
+AWS_ACCESS_KEY_ID="minio"
+AWS_SECRET_ACCESS_KEY="minio123"
+AWS_ENDPOINT="http://localhost:9000"
+INTEGRATION_TEST_S3_BUCKET="test-bucket"

integration_test/src/lib.rs

@@ -0,0 +1,18 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */