From a37f9e6ad04b5fcc69b612593b4ab2c12e5ff964 Mon Sep 17 00:00:00 2001 From: James Bornholt Date: Sun, 4 Feb 2024 17:40:07 -0600 Subject: [PATCH] Update to GA release of Rust SDK Signed-off-by: James Bornholt --- Cargo.lock | 496 +++++++++++------- mountpoint-s3-client/Cargo.toml | 11 +- mountpoint-s3-client/tests/auth.rs | 26 +- mountpoint-s3-client/tests/common/mod.rs | 66 ++- mountpoint-s3-client/tests/delete_object.rs | 28 +- mountpoint-s3-client/tests/endpoint_config.rs | 8 +- mountpoint-s3-client/tests/get_object.rs | 25 +- .../tests/get_object_attributes.rs | 46 +- mountpoint-s3-client/tests/head_object.rs | 13 +- mountpoint-s3-client/tests/imds_crt_client.rs | 4 +- mountpoint-s3-client/tests/put_object.rs | 23 +- mountpoint-s3/Cargo.toml | 7 +- mountpoint-s3/tests/common/fuse.rs | 101 ++-- mountpoint-s3/tests/common/s3.rs | 67 ++- mountpoint-s3/tests/fuse_tests/fork_test.rs | 20 +- 15 files changed, 529 insertions(+), 412 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6150fe88a..fc53559a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -204,132 +204,130 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "0.56.1" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6b3804dca60326e07205179847f17a4fce45af3a1106939177ad41ac08a6de" +checksum = "8b30c39ebe61f75d1b3785362b1586b41991873c9ab3e317a9181c246fb71d82" dependencies = [ "aws-credential-types", - "aws-http", + "aws-runtime", "aws-sdk-sso", + "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", "fastrand", "hex", - "http", + "http 0.2.11", "hyper", - "ring 0.16.20", + "ring", "time", "tokio", - "tower", "tracing", "zeroize", ] [[package]] name = "aws-credential-types" -version = "0.56.1" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a66ac8ef5fa9cf01c2d999f39d16812e90ec1467bd382cbbb74ba23ea86201" +checksum = "33cc49dcdd31c8b6e79850a179af4c367669150c7ac0135f176c61bec81a70f7" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api", "aws-smithy-types", - "fastrand", - "tokio", - "tracing", "zeroize", ] [[package]] -name = "aws-http" -version = "0.56.1" +name = "aws-runtime" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e626370f9ba806ae4c439e49675fd871f5767b093075cdf4fef16cac42ba900" +checksum = "eb031bff99877c26c28895766f7bb8484a05e24547e370768d6cc9db514662aa" dependencies = [ "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "http", + "fastrand", + "http 0.2.11", "http-body", - "lazy_static", "percent-encoding", "pin-project-lite", "tracing", + "uuid", ] [[package]] -name = "aws-runtime" -version = "0.56.1" +name = "aws-sdk-s3" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ac5cf0ff19c1bca0cea7932e11b239d1025a45696a4f44f72ea86e2b8bdd07" +checksum = "951f7730f51a2155c711c85c79f337fbc02a577fa99d2a0a8059acfce5392113" dependencies = [ "aws-credential-types", - "aws-http", + "aws-runtime", "aws-sigv4", "aws-smithy-async", + "aws-smithy-checksums", "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", + "aws-smithy-xml", "aws-types", - "fastrand", - "http", + "bytes", + "http 0.2.11", + "http-body", + "once_cell", "percent-encoding", + "regex-lite", "tracing", - "uuid", + "url", ] [[package]] -name = "aws-sdk-s3" -version = "0.30.0" +name = "aws-sdk-sso" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a531d010f9f556bf65eb3bcd8d24f1937600ab6940fede4d454cd9b1f031fb34" +checksum = "f486420a66caad72635bc2ce0ff6581646e0d32df02aa39dc983bfe794955a5b" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", - "aws-sigv4", "aws-smithy-async", - "aws-smithy-checksums", - "aws-smithy-client", - "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", - "aws-smithy-xml", "aws-types", "bytes", - "http", - "http-body", + "http 0.2.11", "once_cell", - "percent-encoding", - "regex", - "tokio-stream", + "regex-lite", "tracing", - "url", ] [[package]] -name = "aws-sdk-sso" -version = "0.30.0" +name = "aws-sdk-ssooidc" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "903f888ff190e64f6f5c83fb0f8d54f9c20481f1dc26359bb8896f5d99908949" +checksum = "39ddccf01d82fce9b4a15c8ae8608211ee7db8ed13a70b514bbfe41df3d24841" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", "aws-smithy-json", "aws-smithy-runtime", @@ -337,23 +335,21 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "http", - "regex", - "tokio-stream", + "http 0.2.11", + "once_cell", + "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sts" -version = "0.30.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47ad6bf01afc00423d781d464220bf69fb6a674ad6629cbbcb06d88cdc2be82" +checksum = "1a591f8c7e6a621a501b2b5d2e88e1697fcb6274264523a6ad4d5959889a41ce" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", "aws-smithy-json", "aws-smithy-query", @@ -362,49 +358,57 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", - "http", - "regex", + "http 0.2.11", + "once_cell", + "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "0.56.1" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b28f4910bb956b7ab320b62e98096402354eca976c587d1eeccd523d9bac03" +checksum = "c371c6b0ac54d4605eb6f016624fb5c7c2925d315fdf600ac1bf21b19d5f1742" dependencies = [ + "aws-credential-types", "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", "bytes", + "crypto-bigint 0.5.5", "form_urlencoded", "hex", "hmac", - "http", + "http 0.2.11", + "http 1.0.0", "once_cell", + "p256", "percent-encoding", - "regex", + "ring", "sha2", + "subtle", "time", "tracing", + "zeroize", ] [[package]] name = "aws-smithy-async" -version = "0.56.1" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cdb73f85528b9d19c23a496034ac53703955a59323d581c06aa27b4e4e247af" +checksum = "72ee2d09cce0ef3ae526679b522835d63e75fb427aca5413cd371e490d52dcc6" dependencies = [ "futures-util", "pin-project-lite", "tokio", - "tokio-stream", ] [[package]] name = "aws-smithy-checksums" -version = "0.56.1" +version = "0.60.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb15946af1b8d3beeff53ad991d9bff68ac22426b6d40372b958a75fa61eaed" +checksum = "be2acd1b9c6ae5859999250ed5a62423aedc5cf69045b844432de15fa2f31f2b" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -412,7 +416,7 @@ dependencies = [ "crc32c", "crc32fast", "hex", - "http", + "http 0.2.11", "http-body", "md-5", "pin-project-lite", @@ -421,35 +425,11 @@ dependencies = [ "tracing", ] -[[package]] -name = "aws-smithy-client" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c27b2756264c82f830a91cb4d2d485b2d19ad5bea476d9a966e03d27f27ba59a" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "bytes", - "fastrand", - "http", - "http-body", - "hyper", - "hyper-rustls", - "lazy_static", - "pin-project-lite", - "rustls", - "tokio", - "tower", - "tracing", -] - [[package]] name = "aws-smithy-eventstream" -version = "0.56.1" +version = "0.60.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "850233feab37b591b7377fd52063aa37af615687f5896807abe7f49bd4e1d25b" +checksum = "e6363078f927f612b970edf9d1903ef5cef9a64d1e8423525ebb1f0a1633c858" dependencies = [ "aws-smithy-types", "bytes", @@ -458,57 +438,39 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.56.1" +version = "0.60.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54cdcf365d8eee60686885f750a34c190e513677db58bbc466c44c588abf4199" +checksum = "dab56aea3cd9e1101a0a999447fb346afb680ab1406cebc44b32346e25b4117d" dependencies = [ "aws-smithy-eventstream", + "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "bytes-utils", "futures-core", - "http", + "http 0.2.11", "http-body", - "hyper", "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "aws-smithy-http-tower" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822de399d0ce62829a69dfa8c5cd08efdbe61a7426b953e2268f8b8b52a607bd" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "http", - "http-body", - "pin-project-lite", - "tower", "tracing", ] [[package]] name = "aws-smithy-json" -version = "0.56.1" +version = "0.60.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1e7ab8fa7ad10c193af7ae56d2420989e9f4758bf03601a342573333ea34f" +checksum = "fd3898ca6518f9215f62678870064398f00031912390efd03f1f6ef56d83aa8e" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-query" -version = "0.56.1" +version = "0.60.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28556a3902091c1f768a34f6c998028921bdab8d47d92586f363f14a4a32d047" +checksum = "bda4b1dfc9810e35fba8a620e900522cd1bd4f9578c446e82f49d1ce41d2e9f9" dependencies = [ "aws-smithy-types", "urlencoding", @@ -516,76 +478,88 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "0.56.1" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745e096b3553e7e0f40622aa04971ce52765af82bebdeeac53aa6fc82fe801e6" +checksum = "fafdab38f40ad7816e7da5dec279400dd505160780083759f01441af1bbb10ea" dependencies = [ "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "fastrand", - "http", + "h2", + "http 0.2.11", "http-body", + "hyper", + "hyper-rustls", "once_cell", "pin-project-lite", "pin-utils", + "rustls", "tokio", "tracing", ] [[package]] name = "aws-smithy-runtime-api" -version = "0.56.1" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d0ae0c9cfd57944e9711ea610b48a963fb174a53aabacc08c5794a594b1d02" +checksum = "c18276dd28852f34b3bf501f4f3719781f4999a51c7bff1a5c6dc8c4529adc29" dependencies = [ "aws-smithy-async", - "aws-smithy-http", "aws-smithy-types", "bytes", - "http", + "http 0.2.11", + "pin-project-lite", "tokio", "tracing", + "zeroize", ] [[package]] name = "aws-smithy-types" -version = "0.56.1" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d90dbc8da2f6be461fa3c1906b20af8f79d14968fe47f2b7d29d086f62a51728" +checksum = "bb3e134004170d3303718baa2a4eb4ca64ee0a1c0a7041dca31b38be0fb414f3" dependencies = [ "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.11", + "http-body", "itoa", "num-integer", + "pin-project-lite", + "pin-utils", "ryu", "serde", "time", + "tokio", + "tokio-util", ] [[package]] name = "aws-smithy-xml" -version = "0.56.1" +version = "0.60.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01d2dedcdd8023043716cfeeb3c6c59f2d447fce365d8e194838891794b23b6" +checksum = "8604a11b25e9ecaf32f9aa56b9fe253c5e2f606a3477f0071e96d3155a5ed218" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.56.1" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85aa0451bf8af1bf22a4f028d5d28054507a14be43cb8ac0597a8471fba9edfe" +checksum = "789bbe008e65636fe1b6dbbb374c40c8960d1232b96af5ff4aec349f9c4accf4" dependencies = [ "aws-credential-types", "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", - "http", + "http 0.2.11", "rustc_version", "tracing", ] @@ -894,6 +868,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const_format" version = "0.2.32" @@ -1025,6 +1005,28 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -1068,6 +1070,16 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.10" @@ -1100,12 +1112,44 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der", + "elliptic-curve", + "rfc6979", + "signature", +] + [[package]] name = "either" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "env_logger" version = "0.10.1" @@ -1171,6 +1215,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "filetime" version = "0.2.23" @@ -1421,6 +1475,17 @@ dependencies = [ "walkdir", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + [[package]] name = "h2" version = "0.3.24" @@ -1432,7 +1497,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.11", "indexmap", "slab", "tokio", @@ -1520,6 +1585,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -1527,7 +1603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.11", "pin-project-lite", ] @@ -1560,7 +1636,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.11", "http-body", "httparse", "httpdate", @@ -1580,7 +1656,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", + "http 0.2.11", "hyper", "log", "rustls", @@ -1862,6 +1938,7 @@ dependencies = [ "aws-config", "aws-sdk-s3", "aws-sdk-sts", + "aws-smithy-types", "base16ct", "bincode", "built", @@ -1924,6 +2001,7 @@ dependencies = [ "aws-sdk-s3", "aws-sdk-sts", "aws-smithy-runtime-api", + "aws-smithy-types", "base64ct", "built", "bytes", @@ -2114,6 +2192,17 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "page_size" version = "0.6.0" @@ -2197,6 +2286,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.27" @@ -2520,6 +2619,12 @@ dependencies = [ "regex-syntax 0.8.2", ] +[[package]] +name = "regex-lite" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -2533,18 +2638,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] -name = "ring" -version = "0.16.20" +name = "rfc6979" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", + "crypto-bigint 0.4.9", + "hmac", + "zeroize", ] [[package]] @@ -2556,8 +2657,8 @@ dependencies = [ "cc", "getrandom", "libc", - "spin 0.9.8", - "untrusted 0.9.0", + "spin", + "untrusted", "windows-sys 0.48.0", ] @@ -2608,7 +2709,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.17.7", + "ring", "rustls-webpki", "sct", ] @@ -2640,8 +2741,8 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.7", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -2704,8 +2805,22 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.7", - "untrusted 0.9.0", + "ring", + "untrusted", +] + +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", ] [[package]] @@ -2848,6 +2963,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "slab" version = "0.4.9" @@ -2894,15 +3028,19 @@ dependencies = [ [[package]] name = "spin" -version = "0.5.2" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] -name = "spin" -version = "0.9.8" +name = "spki" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der", +] [[package]] name = "static_assertions" @@ -3120,6 +3258,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2 0.5.5", "tokio-macros", "windows-sys 0.48.0", @@ -3146,17 +3285,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.10" @@ -3171,28 +3299,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - [[package]] name = "tower-service" version = "0.3.2" @@ -3306,12 +3412,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" diff --git a/mountpoint-s3-client/Cargo.toml b/mountpoint-s3-client/Cargo.toml index 8b5a16196..d229504f0 100644 --- a/mountpoint-s3-client/Cargo.toml +++ b/mountpoint-s3-client/Cargo.toml @@ -39,11 +39,12 @@ rand_chacha = { version = "0.3.1", optional = true } [dev-dependencies] anyhow = { version = "1.0.64", features = ["backtrace"] } -aws-config = "0.56.0" -aws-credential-types = "0.56.0" -aws-sdk-s3 = "0.30.0" -aws-sdk-sts = "0.30.0" -aws-smithy-runtime-api = "0.56.1" +aws-config = "1.1.4" +aws-credential-types = "1.1.4" +aws-sdk-s3 = "1.14.0" +aws-sdk-sts = "1.12.0" +aws-smithy-runtime-api = "1.1.4" +aws-smithy-types = "1.1.4" bytes = "1.2.1" clap = { version = "4.1.9", features = ["derive"] } ctor = "0.2.6" diff --git a/mountpoint-s3-client/tests/auth.rs b/mountpoint-s3-client/tests/auth.rs index 007a36a22..1288567df 100644 --- a/mountpoint-s3-client/tests/auth.rs +++ b/mountpoint-s3-client/tests/auth.rs @@ -6,6 +6,8 @@ use std::io::Write; use std::option::Option::None; use aws_config::default_provider::credentials::DefaultCredentialsChain; +#[cfg(not(feature = "s3express_tests"))] +use aws_config::BehaviorVersion; use aws_credential_types::provider::ProvideCredentials; use aws_sdk_s3::config::Region; use aws_sdk_s3::primitives::ByteStream; @@ -31,11 +33,9 @@ async fn test_static_provider() { let key = format!("{prefix}/hello"); let body = b"hello world!"; - let mut request = sdk_client.put_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - request + sdk_client + .put_object() + .bucket(&bucket) .key(&key) .body(ByteStream::from(Bytes::from_static(body))) .send() @@ -108,11 +108,9 @@ async fn test_profile_provider_async() { let key = format!("{prefix}/hello"); let body = b"hello world!"; - let mut request = sdk_client.put_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - request + sdk_client + .put_object() + .bucket(&bucket) .key(&key) .body(ByteStream::from(Bytes::from_static(body))) .send() @@ -220,7 +218,7 @@ async fn test_scoped_credentials() { .unwrap(); } - let config = aws_config::from_env() + let config = aws_config::defaults(BehaviorVersion::latest()) .region(Region::new(get_test_region())) .load() .await; @@ -246,9 +244,9 @@ async fn test_scoped_credentials() { // Build a S3CrtClient that uses a static credentials provider with the creds we just got let config = CredentialsProviderStaticOptions { - access_key_id: credentials.access_key_id().unwrap(), - secret_access_key: credentials.secret_access_key().unwrap(), - session_token: credentials.session_token(), + access_key_id: credentials.access_key_id(), + secret_access_key: credentials.secret_access_key(), + session_token: Some(credentials.session_token()), }; let provider = CredentialsProvider::new_static(&Allocator::default(), config).unwrap(); let config = S3ClientConfig::new() diff --git a/mountpoint-s3-client/tests/common/mod.rs b/mountpoint-s3-client/tests/common/mod.rs index 768690ee8..3912969f0 100644 --- a/mountpoint-s3-client/tests/common/mod.rs +++ b/mountpoint-s3-client/tests/common/mod.rs @@ -1,5 +1,6 @@ #![cfg(feature = "s3_tests")] +use aws_config::BehaviorVersion; use aws_sdk_s3 as s3; use aws_sdk_s3::config::Region; use aws_sdk_s3::error::SdkError; @@ -114,23 +115,54 @@ pub fn get_s3express_endpoint() -> String { } pub async fn get_test_sdk_client() -> s3::Client { - let mut config = aws_config::from_env().region(Region::new(get_test_region())); + let sdk_config = aws_config::defaults(BehaviorVersion::latest()) + .region(Region::new(get_test_region())) + .load() + .await; + let mut s3_config = s3::config::Builder::from(&sdk_config); + + // TODO: remove when the Rust SDK supports S3 Express One Zone. For now, we force the SDK to + // always use SigV4 and point it at the manually specified Express One Zone endpoint. if cfg!(feature = "s3express_tests") { - config = config.endpoint_url(get_s3express_endpoint()); + let endpoint = s3::config::endpoint::Endpoint::builder() + .url(get_s3express_endpoint()) + .property( + "authSchemes", + vec![aws_smithy_types::Document::from({ + let mut out = std::collections::HashMap::::new(); + out.insert("disableDoubleEncoding".to_string(), true.into()); + out.insert("name".to_string(), "sigv4".to_string().into()); + out.insert("signingName".to_string(), "s3".to_string().into()); + out.insert("signingRegion".to_string(), get_test_region().into()); + out + })], + ) + .build(); + + #[derive(Debug)] + struct StaticEndpointResolver(aws_sdk_s3::config::endpoint::Endpoint); + + impl s3::config::endpoint::ResolveEndpoint for StaticEndpointResolver { + fn resolve_endpoint( + &self, + _params: &s3::config::endpoint::Params, + ) -> s3::config::endpoint::EndpointFuture<'_> { + s3::config::endpoint::EndpointFuture::ready(Ok(self.0.clone())) + } + } + + s3_config = s3_config.endpoint_resolver(StaticEndpointResolver(endpoint)); } - let config = config.load().await; - s3::Client::new(&config) + + s3::Client::from_conf(s3_config.build()) } /// Create some objects in a prefix for testing. pub async fn create_objects_for_test(client: &s3::Client, bucket: &str, prefix: &str, names: &[impl AsRef]) { for name in names { - let mut request = client.put_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(bucket); - } - - request + client + .put_object() + .bucket(bucket) .key(format!("{}{}", prefix, name.as_ref())) .body(ByteStream::from(Bytes::from_static(b"."))) .send() @@ -145,20 +177,18 @@ pub async fn get_mpu_count_for_key( prefix: &str, key: &str, ) -> Result> { - let mut request = client.list_multipart_uploads(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(bucket); - } - // This could be broken if we have initiated more than one multipart upload using the same key // since ListMultipartUploads returns all multipart uploads for that key. - let upload_count = request + let upload_count = client + .list_multipart_uploads() + .bucket(bucket) .prefix(prefix) .send() .await? .uploads() - .map(|upload| upload.iter().filter(|&u| u.key() == Some(key)).collect::>()) - .map_or(0, |u| u.len()); + .iter() + .filter(|&u| u.key() == Some(key)) + .count(); Ok(upload_count) } diff --git a/mountpoint-s3-client/tests/delete_object.rs b/mountpoint-s3-client/tests/delete_object.rs index a88d519ee..eb5acad2f 100644 --- a/mountpoint-s3-client/tests/delete_object.rs +++ b/mountpoint-s3-client/tests/delete_object.rs @@ -15,11 +15,9 @@ async fn test_delete_object() { let key = format!("{prefix}/hello"); let body = b"hello world!"; - let mut request = sdk_client.put_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - request + sdk_client + .put_object() + .bucket(&bucket) .key(&key) .body(ByteStream::from(Bytes::from_static(body))) .send() @@ -32,11 +30,13 @@ async fn test_delete_object() { .await .expect("delete_object should succeed"); - let mut request = sdk_client.head_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - let head_obj_err = request.key(&key).send().await.expect_err("object should not exist"); + let head_obj_err = sdk_client + .head_object() + .bucket(&bucket) + .key(&key) + .send() + .await + .expect_err("object should not exist"); assert!(head_obj_err.into_service_error().is_not_found()); } @@ -48,11 +48,9 @@ async fn test_delete_object_no_obj() { let key = format!("{prefix}/nonexistent_key"); - let mut request = sdk_client.head_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - let head_obj_err = request + let head_obj_err = sdk_client + .head_object() + .bucket(&bucket) .bucket(&bucket) .key(&key) .send() diff --git a/mountpoint-s3-client/tests/endpoint_config.rs b/mountpoint-s3-client/tests/endpoint_config.rs index 9bbc8120a..aae6deff7 100644 --- a/mountpoint-s3-client/tests/endpoint_config.rs +++ b/mountpoint-s3-client/tests/endpoint_config.rs @@ -15,11 +15,9 @@ async fn run_test EndpointConfig>(f: F, prefix: &str, bucket: // Create one object named "hello" let key = format!("{prefix}hello"); let body = b"hello world!"; - let mut request = sdk_client.put_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - request + sdk_client + .put_object() + .bucket(&bucket) .key(&key) .body(ByteStream::from(Bytes::from_static(body))) .send() diff --git a/mountpoint-s3-client/tests/get_object.rs b/mountpoint-s3-client/tests/get_object.rs index b139017cf..472facb44 100644 --- a/mountpoint-s3-client/tests/get_object.rs +++ b/mountpoint-s3-client/tests/get_object.rs @@ -30,11 +30,9 @@ async fn test_get_object(size: usize, range: Option>) { let key = format!("{prefix}/test"); let body = vec![0x42; size]; - let mut request = sdk_client.put_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - request + sdk_client + .put_object() + .bucket(&bucket) .key(&key) .body(ByteStream::from(body.clone())) .send() @@ -102,11 +100,10 @@ async fn test_get_object_success_if_match() { // Create one object named "hello" let key = format!("{prefix}/hello"); let body = b"hello world!"; - let mut request = sdk_client.put_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - let response = request + + let response = sdk_client + .put_object() + .bucket(&bucket) .key(&key) .body(ByteStream::from(Bytes::from_static(body))) .send() @@ -131,11 +128,9 @@ async fn test_get_object_412_if_match() { // Create one object named "hello" let key = format!("{prefix}/hello"); let body = b"hello world!"; - let mut request = sdk_client.put_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - request + sdk_client + .put_object() + .bucket(&bucket) .key(&key) .body(ByteStream::from(Bytes::from_static(body))) .send() diff --git a/mountpoint-s3-client/tests/get_object_attributes.rs b/mountpoint-s3-client/tests/get_object_attributes.rs index f17fa68b5..a432bde3c 100644 --- a/mountpoint-s3-client/tests/get_object_attributes.rs +++ b/mountpoint-s3-client/tests/get_object_attributes.rs @@ -28,11 +28,7 @@ async fn create_mpu_object( ) -> (Vec, CompleteMultipartUploadOutput) { let sdk_client = get_test_sdk_client().await; - let mut request = sdk_client.create_multipart_upload(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(bucket); - } - let mut create_mpu = request.key(key); + let mut create_mpu = sdk_client.create_multipart_upload().bucket(bucket).key(key); if let Some(algorithm) = &checksum_algorithm { create_mpu = create_mpu.checksum_algorithm(algorithm.to_owned()); } @@ -46,11 +42,9 @@ async fn create_mpu_object( let part_body = vec![0; part_size.to_owned()]; let part_num = (part_index + 1) as i32; - let mut request = sdk_client.upload_part(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(bucket); - } - let mut upload_part = request + let mut upload_part = sdk_client + .upload_part() + .bucket(bucket) .key(key) .part_number(part_num) .upload_id(upload_id) @@ -92,11 +86,9 @@ async fn create_mpu_object( .set_parts(Some(completed_parts.clone())) .build(); - let mut request = sdk_client.complete_multipart_upload(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(bucket); - } - let complete_mpu_output = request + let complete_mpu_output = sdk_client + .complete_multipart_upload() + .bucket(bucket) .key(key) .upload_id(upload_id) .multipart_upload(completed_mpu) @@ -115,11 +107,9 @@ async fn test_with_checksum(checksum_algorithm: ChecksumAlgorithm) { let key = format!("{prefix}/hello"); let body = b"hello world!"; - let mut request = sdk_client.put_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - let put_object_output = request + let put_object_output = sdk_client + .put_object() + .bucket(&bucket) .key(&key) .checksum_algorithm(checksum_algorithm.clone()) .body(ByteStream::from(Bytes::from_static(body))) @@ -181,11 +171,9 @@ async fn test_get_attributes() { // Create one object named "hello" let key = format!("{prefix}/hello"); let body = b"hello world!"; - let mut request = sdk_client.put_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - let put_object_output = request + let put_object_output = sdk_client + .put_object() + .bucket(&bucket) .key(&key) .body(ByteStream::from(Bytes::from_static(body))) .send() @@ -234,11 +222,9 @@ async fn test_get_attributes_all_none() { // Create one object named "hello" let key = format!("{prefix}/hello"); let body = b"hello world!"; - let mut request = sdk_client.put_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - request + sdk_client + .put_object() + .bucket(&bucket) .key(&key) .checksum_algorithm(ChecksumAlgorithm::Crc32C) .body(ByteStream::from(Bytes::from_static(body))) diff --git a/mountpoint-s3-client/tests/head_object.rs b/mountpoint-s3-client/tests/head_object.rs index e7c825bad..77a9173fa 100644 --- a/mountpoint-s3-client/tests/head_object.rs +++ b/mountpoint-s3-client/tests/head_object.rs @@ -24,11 +24,9 @@ async fn test_head_object() { let key = format!("{prefix}/hello"); let body = b"hello world!"; - let mut request = sdk_client.put_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - request + sdk_client + .put_object() + .bucket(&bucket) .key(&key) .body(ByteStream::from(Bytes::from_static(body))) .send() @@ -157,7 +155,10 @@ async fn test_head_object_restored() { RestoreRequest::builder() .set_days(Some(1)) .set_glacier_job_parameters(Some( - GlacierJobParameters::builder().set_tier(Some(Tier::Expedited)).build(), + GlacierJobParameters::builder() + .set_tier(Some(Tier::Expedited)) + .build() + .unwrap(), )) .build(), ) diff --git a/mountpoint-s3-client/tests/imds_crt_client.rs b/mountpoint-s3-client/tests/imds_crt_client.rs index a73fdb304..507d17a2b 100644 --- a/mountpoint-s3-client/tests/imds_crt_client.rs +++ b/mountpoint-s3-client/tests/imds_crt_client.rs @@ -6,10 +6,10 @@ use serde_json::Value; #[tokio::test] async fn test_get_identity_document() { - let sdk_client = Client::builder().build().await.unwrap(); + let sdk_client = Client::builder().build(); match sdk_client.get("/latest/dynamic/instance-identity/document").await { Ok(expected_json) => { - let expected_doc: Value = serde_json::from_str(&expected_json).unwrap(); + let expected_doc: Value = serde_json::from_str(expected_json.as_ref()).unwrap(); let expected_region = expected_doc.get("region").unwrap().as_str().unwrap(); let expected_instance_type = expected_doc.get("instanceType").unwrap().as_str().unwrap(); diff --git a/mountpoint-s3-client/tests/put_object.rs b/mountpoint-s3-client/tests/put_object.rs index 417a231e8..e1e4899ba 100644 --- a/mountpoint-s3-client/tests/put_object.rs +++ b/mountpoint-s3-client/tests/put_object.rs @@ -200,17 +200,15 @@ async fn test_put_checksums() { request.complete().await.unwrap(); let sdk_client = get_test_sdk_client().await; - let mut request = sdk_client.get_object_attributes(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&bucket); - } - let attributes: aws_sdk_s3::operation::get_object_attributes::GetObjectAttributesOutput = request + let attributes = sdk_client + .get_object_attributes() + .bucket(&bucket) .key(key) .object_attributes(aws_sdk_s3::types::ObjectAttributes::ObjectParts) .send() .await .unwrap(); - let parts = attributes.object_parts().unwrap().parts().unwrap(); + let parts = attributes.object_parts().unwrap().parts(); let checksums: Vec<_> = parts.iter().map(|p| p.checksum_crc32_c().unwrap()).collect(); let expected_checksums: Vec<_> = contents.chunks(PART_SIZE).map(crc32c::checksum).collect(); @@ -329,12 +327,13 @@ async fn test_put_object_storage_class(storage_class: &str) { #[cfg(not(feature = "s3express_tests"))] async fn check_sse(bucket: &String, key: &String, expected_sse: Option<&str>, expected_key: &Option) { let sdk_client = get_test_sdk_client().await; - let mut request = sdk_client.head_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(bucket); - } - let head_object_resp: aws_sdk_s3::operation::head_object::HeadObjectOutput = - request.key(key).send().await.expect("head object should succeed"); + let head_object_resp = sdk_client + .head_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect("head object should succeed"); let expected_sse = match expected_sse { None => aws_sdk_s3::types::ServerSideEncryption::Aes256, Some("aws:kms") => aws_sdk_s3::types::ServerSideEncryption::AwsKms, diff --git a/mountpoint-s3/Cargo.toml b/mountpoint-s3/Cargo.toml index 7a784c1b5..30daed730 100644 --- a/mountpoint-s3/Cargo.toml +++ b/mountpoint-s3/Cargo.toml @@ -49,9 +49,10 @@ mountpoint-s3-client = { path = "../mountpoint-s3-client", features = ["mock"] } assert_cmd = "2.0.6" assert_fs = "1.1.1" -aws-config = "0.56.0" -aws-sdk-s3 = "0.30.0" -aws-sdk-sts = "0.30.0" +aws-config = "1.1.4" +aws-sdk-s3 = "1.14.0" +aws-sdk-sts = "1.12.0" +aws-smithy-types = "1.1.4" base16ct = { version = "0.1.1", features = ["alloc"] } ctor = "0.2.6" filetime = "0.2.21" diff --git a/mountpoint-s3/tests/common/fuse.rs b/mountpoint-s3/tests/common/fuse.rs index 74e25633b..63648f8c9 100644 --- a/mountpoint-s3/tests/common/fuse.rs +++ b/mountpoint-s3/tests/common/fuse.rs @@ -243,7 +243,6 @@ pub mod mock_session { pub mod s3_session { use super::*; - use aws_sdk_s3::config::Region; use aws_sdk_s3::operation::head_object::HeadObjectError; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{ChecksumAlgorithm, GlacierJobParameters, RestoreRequest, Tier}; @@ -252,7 +251,7 @@ pub mod s3_session { use mountpoint_s3_client::config::{EndpointConfig, S3ClientConfig}; use mountpoint_s3_client::S3CrtClient; - use crate::common::s3::{get_s3express_endpoint, get_test_bucket_and_prefix, get_test_region, tokio_block_on}; + use crate::common::s3::{get_test_bucket_and_prefix, get_test_region, get_test_sdk_client, tokio_block_on}; /// Create a FUSE mount backed by a real S3 client pub fn new(test_name: &str, test_config: TestSessionConfig) -> (TempDir, BackgroundSession, TestClientBox) { @@ -324,15 +323,6 @@ pub mod s3_session { Box::new(test_client) } - async fn get_test_sdk_client(region: &str) -> aws_sdk_s3::Client { - let mut config = aws_config::from_env().region(Region::new(region.to_owned())); - if cfg!(feature = "s3express_tests") { - config = config.endpoint_url(get_s3express_endpoint()); - } - let config = config.load().await; - aws_sdk_s3::Client::new(&config) - } - struct SDKTestClient { prefix: String, bucket: String, @@ -350,13 +340,10 @@ pub mod s3_session { let mut request = self .sdk_client .put_object() + .bucket(&self.bucket) .key(full_key) .body(ByteStream::from(value.to_vec())); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&self.bucket); - } - if let Some(storage_class) = params.storage_class { request = request.set_storage_class(Some(storage_class.as_str().into())); } @@ -370,40 +357,35 @@ pub mod s3_session { fn remove_object(&mut self, key: &str) -> Result<(), Box> { let full_key = format!("{}{}", self.prefix, key); - let mut request = self.sdk_client.delete_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&self.bucket); - } - tokio_block_on(request.key(full_key).send()) - .map(|_| ()) - .map_err(|e| Box::new(e) as Box) + tokio_block_on( + self.sdk_client + .delete_object() + .bucket(&self.bucket) + .key(full_key) + .send(), + ) + .map(|_| ()) + .map_err(|e| Box::new(e) as Box) } fn contains_dir(&self, key: &str) -> Result> { let full_key_suffixed = format!("{}{}/", self.prefix, key); - let mut request = self.sdk_client.list_objects_v2(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&self.bucket); - } - - tokio_block_on(request.delimiter('/').prefix(full_key_suffixed).send()) - .map(|output| { - let len = output.contents().map(|c| c.len()).unwrap_or_default() - + output.common_prefixes().map(|c| c.len()).unwrap_or_default(); - len > 0 - }) - .map_err(|e| Box::new(e) as Box) + tokio_block_on( + self.sdk_client + .list_objects_v2() + .bucket(&self.bucket) + .delimiter('/') + .prefix(full_key_suffixed) + .send(), + ) + .map(|output| !(output.contents().is_empty() && output.common_prefixes().is_empty())) + .map_err(|e| Box::new(e) as Box) } fn contains_key(&self, key: &str) -> Result> { let full_key = format!("{}{}", self.prefix, key); - let mut request = self.sdk_client.head_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&self.bucket); - } - - let result = tokio_block_on(request.key(full_key).send()); + let result = tokio_block_on(self.sdk_client.head_object().bucket(&self.bucket).key(full_key).send()); match result { Ok(_) => Ok(true), Err(e) => match e.into_service_error() { @@ -414,36 +396,23 @@ pub mod s3_session { } fn is_upload_in_progress(&self, key: &str) -> Result> { - let mut request = self.sdk_client.list_multipart_uploads(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&self.bucket); - } - - tokio_block_on(request.prefix(self.prefix.clone()).send()) - .map(|output| { - output - .uploads() - .map(|upload| { - upload - .iter() - .filter(|&u| u.key().unwrap().ends_with(key)) - .collect::>() - }) - .map_or(0, |u| u.len()) - > 0 - }) - .map_err(|e| Box::new(e) as Box) + tokio_block_on( + self.sdk_client + .list_multipart_uploads() + .bucket(&self.bucket) + .prefix(self.prefix.clone()) + .send(), + ) + .map(|output| output.uploads().iter().any(|u| u.key().unwrap().ends_with(key))) + .map_err(|e| Box::new(e) as Box) } fn get_object_storage_class(&self, key: &str) -> Result, Box> { let full_key = format!("{}{}", self.prefix, key); - let mut request = self.sdk_client.get_object_attributes(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(&self.bucket); - } - tokio_block_on( - request + self.sdk_client + .get_object_attributes() + .bucket(&self.bucket) .key(full_key) .object_attributes(aws_sdk_s3::types::ObjectAttributes::StorageClass) .send(), @@ -466,7 +435,7 @@ pub mod s3_session { RestoreRequest::builder() .set_days(Some(1)) .set_glacier_job_parameters(Some( - GlacierJobParameters::builder().set_tier(Some(tier)).build(), + GlacierJobParameters::builder().set_tier(Some(tier)).build()?, )) .build(), )) diff --git a/mountpoint-s3/tests/common/s3.rs b/mountpoint-s3/tests/common/s3.rs index 0fb008ef4..8c7333df6 100644 --- a/mountpoint-s3/tests/common/s3.rs +++ b/mountpoint-s3/tests/common/s3.rs @@ -1,5 +1,5 @@ +use aws_config::{BehaviorVersion, Region}; use aws_sdk_s3::primitives::ByteStream; -use aws_sdk_sts::config::Region; use futures::Future; use rand::RngCore; use rand_chacha::rand_core::OsRng; @@ -40,26 +40,61 @@ pub fn get_s3express_endpoint() -> String { std::env::var("S3_EXPRESS_ONE_ZONE_ENDPOINT").expect("Set S3_EXPRESS_ONE_ZONE_ENDPOINT to run integration tests") } -pub fn create_objects(bucket: &str, prefix: &str, region: &str, key: &str, value: &[u8]) { - let mut config = aws_config::from_env().region(Region::new(region.to_string())); +pub async fn get_test_sdk_client(region: &str) -> aws_sdk_s3::Client { + let sdk_config = aws_config::defaults(BehaviorVersion::latest()) + .region(Region::new(get_test_region())) + .load() + .await; + let mut s3_config = aws_sdk_s3::config::Builder::from(&sdk_config); + + // TODO: remove when the Rust SDK supports S3 Express One Zone. For now, we force the SDK to + // always use SigV4 and point it at the manually specified Express One Zone endpoint. if cfg!(feature = "s3express_tests") { - config = config.endpoint_url(get_s3express_endpoint()); + let endpoint = aws_sdk_s3::config::endpoint::Endpoint::builder() + .url(get_s3express_endpoint()) + .property( + "authSchemes", + vec![aws_smithy_types::Document::from({ + let mut out = std::collections::HashMap::::new(); + out.insert("disableDoubleEncoding".to_string(), true.into()); + out.insert("name".to_string(), "sigv4".to_string().into()); + out.insert("signingName".to_string(), "s3".to_string().into()); + out.insert("signingRegion".to_string(), region.to_string().into()); + out + })], + ) + .build(); + + #[derive(Debug)] + struct StaticEndpointResolver(aws_sdk_s3::config::endpoint::Endpoint); + + impl aws_sdk_s3::config::endpoint::ResolveEndpoint for StaticEndpointResolver { + fn resolve_endpoint( + &self, + _params: &aws_sdk_s3::config::endpoint::Params, + ) -> aws_sdk_s3::config::endpoint::EndpointFuture<'_> { + aws_sdk_s3::config::endpoint::EndpointFuture::ready(Ok(self.0.clone())) + } + } + + s3_config = s3_config.endpoint_resolver(StaticEndpointResolver(endpoint)); } - let config = tokio_block_on(config.load()); - let sdk_client = aws_sdk_s3::Client::new(&config); + + aws_sdk_s3::Client::from_conf(s3_config.build()) +} + +pub fn create_objects(bucket: &str, prefix: &str, region: &str, key: &str, value: &[u8]) { + let sdk_client = tokio_block_on(get_test_sdk_client(region)); let full_key = format!("{prefix}{key}"); - tokio_block_on(async move { - let mut request = sdk_client.put_object(); - if cfg!(not(feature = "s3express_tests")) { - request = request.bucket(bucket); - } - request + tokio_block_on( + sdk_client + .put_object() + .bucket(bucket) .key(full_key) .body(ByteStream::from(value.to_vec())) - .send() - .await - .unwrap() - }); + .send(), + ) + .unwrap(); } pub fn tokio_block_on(future: F) -> F::Output { diff --git a/mountpoint-s3/tests/fuse_tests/fork_test.rs b/mountpoint-s3/tests/fuse_tests/fork_test.rs index 095812af4..212f7b72d 100644 --- a/mountpoint-s3/tests/fuse_tests/fork_test.rs +++ b/mountpoint-s3/tests/fuse_tests/fork_test.rs @@ -3,6 +3,8 @@ use assert_cmd::prelude::*; #[cfg(not(feature = "s3express_tests"))] +use aws_config::BehaviorVersion; +#[cfg(not(feature = "s3express_tests"))] use aws_sdk_sts::config::Region; use std::fs; use std::io::{BufRead, BufReader}; @@ -320,7 +322,11 @@ fn mount_scoped_credentials() -> Result<(), Box> { {"Effect": "Allow", "Action": "s3:ListBucket", "Resource": "arn:aws:s3:::__BUCKET__", "Condition": {"StringLike": {"s3:prefix": "__PREFIX__*"}}} ]}"#; let policy = policy.replace("__BUCKET__", &bucket).replace("__PREFIX__", &subprefix); - let config = tokio_block_on(aws_config::from_env().region(Region::new(get_test_region())).load()); + let config = tokio_block_on( + aws_config::defaults(BehaviorVersion::latest()) + .region(Region::new(get_test_region())) + .load(), + ); let sts_client = aws_sdk_sts::Client::new(&config); let credentials = tokio_block_on( sts_client @@ -341,9 +347,9 @@ fn mount_scoped_credentials() -> Result<(), Box> { .arg(format!("--prefix={prefix}")) .arg("--auto-unmount") .arg(format!("--region={region}")) - .env("AWS_ACCESS_KEY_ID", credentials.access_key_id().unwrap()) - .env("AWS_SECRET_ACCESS_KEY", credentials.secret_access_key().unwrap()) - .env("AWS_SESSION_TOKEN", credentials.session_token().unwrap()) + .env("AWS_ACCESS_KEY_ID", credentials.access_key_id()) + .env("AWS_SECRET_ACCESS_KEY", credentials.secret_access_key()) + .env("AWS_SESSION_TOKEN", credentials.session_token()) .spawn() .expect("unable to spawn child"); @@ -361,9 +367,9 @@ fn mount_scoped_credentials() -> Result<(), Box> { .arg(format!("--prefix={subprefix}")) .arg("--auto-unmount") .arg(format!("--region={region}")) - .env("AWS_ACCESS_KEY_ID", credentials.access_key_id().unwrap()) - .env("AWS_SECRET_ACCESS_KEY", credentials.secret_access_key().unwrap()) - .env("AWS_SESSION_TOKEN", credentials.session_token().unwrap()) + .env("AWS_ACCESS_KEY_ID", credentials.access_key_id()) + .env("AWS_SECRET_ACCESS_KEY", credentials.secret_access_key()) + .env("AWS_SESSION_TOKEN", credentials.session_token()) .spawn() .expect("unable to spawn child");