From d7558d4550e6f68a34d74212caac3ec60af25ae4 Mon Sep 17 00:00:00 2001 From: amrltqt Date: Mon, 22 Aug 2022 15:04:33 +0200 Subject: [PATCH 1/2] update variable name for amazonS3builder in integration (#2550) --- .github/workflows/object_store.yml | 8 ++++---- object_store/src/aws/mod.rs | 33 ++++++++++++++++-------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index 5da2cb4e6cba..bd2f08755942 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -81,10 +81,10 @@ jobs: RUST_BACKTRACE: "1" # Run integration tests TEST_INTEGRATION: 1 - AWS_DEFAULT_REGION: "us-east-1" - AWS_ACCESS_KEY_ID: test - AWS_SECRET_ACCESS_KEY: test - AWS_ENDPOINT: http://localstack:4566 + OBJECT_STORE_AWS_DEFAULT_REGION: "us-east-1" + OBJECT_STORE_AWS_ACCESS_KEY_ID: test + OBJECT_STORE_AWS_SECRET_ACCESS_KEY: test + OBJECT_STORE_AWS_ENDPOINT: http://localstack:4566 EC2_METADATA_ENDPOINT: http://ec2-metadata:1338 AZURE_USE_EMULATOR: "1" AZURITE_BLOB_STORAGE_URL: "http://azurite:10000" diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs index 2a13083463b3..9a7a5b84f2d3 100644 --- a/object_store/src/aws/mod.rs +++ b/object_store/src/aws/mod.rs @@ -547,10 +547,10 @@ mod tests { dotenv::dotenv().ok(); let required_vars = [ - "AWS_DEFAULT_REGION", + "OBJECT_STORE_AWS_DEFAULT_REGION", "OBJECT_STORE_BUCKET", - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", + "OBJECT_STORE_AWS_ACCESS_KEY_ID", + "OBJECT_STORE_AWS_SECRET_ACCESS_KEY", ]; let unset_vars: Vec<_> = required_vars .iter() @@ -582,16 +582,16 @@ mod tests { } else { let config = AmazonS3Builder::new() .with_access_key_id( - env::var("AWS_ACCESS_KEY_ID") - .expect("already checked AWS_ACCESS_KEY_ID"), + env::var("OBJECT_STORE_AWS_ACCESS_KEY_ID") + .expect("already checked OBJECT_STORE_AWS_ACCESS_KEY_ID"), ) .with_secret_access_key( - env::var("AWS_SECRET_ACCESS_KEY") - .expect("already checked AWS_SECRET_ACCESS_KEY"), + env::var("OBJECT_STORE_AWS_SECRET_ACCESS_KEY") + .expect("already checked OBJECT_STORE_AWS_SECRET_ACCESS_KEY"), ) .with_region( - env::var("AWS_DEFAULT_REGION") - .expect("already checked AWS_DEFAULT_REGION"), + env::var("OBJECT_STORE_AWS_DEFAULT_REGION") + .expect("already checked OBJECT_STORE_AWS_DEFAULT_REGION"), ) .with_bucket_name( env::var("OBJECT_STORE_BUCKET") @@ -599,13 +599,16 @@ mod tests { ) .with_allow_http(true); - let config = if let Some(endpoint) = env::var("AWS_ENDPOINT").ok() { - config.with_endpoint(endpoint) - } else { - config - }; + let config = + if let Some(endpoint) = env::var("OBJECT_STORE_AWS_ENDPOINT").ok() { + config.with_endpoint(endpoint) + } else { + config + }; - let config = if let Some(token) = env::var("AWS_SESSION_TOKEN").ok() { + let config = if let Some(token) = + env::var("OBJECT_STORE_AWS_SESSION_TOKEN").ok() + { config.with_token(token) } else { config From 81ad026a00914cb86c1d75341a56abe4eaffa2d6 Mon Sep 17 00:00:00 2001 From: amrltqt Date: Mon, 22 Aug 2022 19:02:26 +0200 Subject: [PATCH 2/2] apply env at step level (#2550) --- .github/workflows/object_store.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index bd2f08755942..cc330807114e 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -81,10 +81,6 @@ jobs: RUST_BACKTRACE: "1" # Run integration tests TEST_INTEGRATION: 1 - OBJECT_STORE_AWS_DEFAULT_REGION: "us-east-1" - OBJECT_STORE_AWS_ACCESS_KEY_ID: test - OBJECT_STORE_AWS_SECRET_ACCESS_KEY: test - OBJECT_STORE_AWS_ENDPOINT: http://localstack:4566 EC2_METADATA_ENDPOINT: http://ec2-metadata:1338 AZURE_USE_EMULATOR: "1" AZURITE_BLOB_STORAGE_URL: "http://azurite:10000" @@ -101,6 +97,11 @@ jobs: echo '{"gcs_base_url": "https://fake-gcs:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT" - name: Setup LocalStack (AWS emulation) + env: + AWS_DEFAULT_REGION: "us-east-1" + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_ENDPOINT: http://localstack:4566 run: | cd /tmp curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" @@ -121,6 +122,11 @@ jobs: rustup default stable - name: Run object_store tests + env: + OBJECT_STORE_AWS_DEFAULT_REGION: "us-east-1" + OBJECT_STORE_AWS_ACCESS_KEY_ID: test + OBJECT_STORE_AWS_SECRET_ACCESS_KEY: test + OBJECT_STORE_AWS_ENDPOINT: http://localstack:4566 run: | # run tests cargo test -p object_store --features=aws,azure,azure_test,gcp