From b6344d2d0dcda57ed4952952afba0761eccafb1b Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 7 Jun 2023 12:30:51 -0600 Subject: [PATCH] RUST-906 Add native support for AWS IAM Roles for service accounts, EKS in particular (#885) --- .evergreen/config.yml | 53 +++++++++- src/client/auth/aws.rs | 163 ++++++++++++++++++++---------- src/client/csfle/state_machine.rs | 28 ++--- src/runtime/http.rs | 110 +++++++++----------- 4 files changed, 216 insertions(+), 138 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d8d9d7ab5..dabd31439 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -127,7 +127,12 @@ functions: "iam_auth_assume_role_name" : "${iam_auth_assume_role_name}", "iam_auth_ec2_instance_account" : "${iam_auth_ec2_instance_account}", "iam_auth_ec2_instance_secret_access_key" : "${iam_auth_ec2_instance_secret_access_key}", - "iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}" + "iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}", + "iam_auth_assume_web_role_name": "${iam_auth_assume_web_role_name}", + "iam_web_identity_issuer": "${iam_web_identity_issuer}", + "iam_web_identity_jwks_uri": "${iam_web_identity_jwks_uri}", + "iam_web_identity_token_file": "${iam_web_identity_token_file}", + "iam_web_identity_rsa_key": "${iam_web_identity_rsa_key}" } EOF @@ -297,6 +302,46 @@ functions: cat setup.js mongo --nodb setup.js aws_e2e_ecs.js + "run aws assume role with web identity test": + - command: shell.exec + type: test + params: + shell: bash + working_dir: "src" + script: | + ${PREPARE_SHELL} + cd ${DRIVERS_TOOLS}/.evergreen/auth_aws + . ./activate-authawsvenv.sh + mongo aws_e2e_web_identity.js + - command: shell.exec + type: test + params: + working_dir: "src" + silent: true + script: | + # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) + cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" + export AWS_ROLE_ARN="${iam_auth_assume_web_role_name}" + export AWS_WEB_IDENTITY_TOKEN_FILE="${iam_web_identity_token_file}" + EOF + - command: shell.exec + type: test + params: + shell: bash + working_dir: "src" + script: | + # the test should be run with and without a session name set + ASYNC_RUNTIME=${ASYNC_RUNTIME} \ + PROJECT_DIRECTORY=${PROJECT_DIRECTORY} \ + ASSERT_NO_URI_CREDS=true \ + AWS_ROLE_SESSION_NAME="test" \ + .evergreen/run-aws-tests.sh + ASYNC_RUNTIME=${ASYNC_RUNTIME} \ + PROJECT_DIRECTORY=${PROJECT_DIRECTORY} \ + ASSERT_NO_URI_CREDS=true \ + .evergreen/run-aws-tests.sh + + "run x509 tests": - command: shell.exec type: test @@ -1023,6 +1068,7 @@ tasks: - func: "run aws auth test with aws credentials and session token as environment variables" - func: "run aws auth test with aws EC2 credentials" - func: "run aws ECS auth test" + - func: "run aws assume role with web identity test" - name: "test-5.0-standalone" tags: ["5.0", "standalone"] @@ -1083,6 +1129,7 @@ tasks: - func: "run aws auth test with aws credentials and session token as environment variables" - func: "run aws auth test with aws EC2 credentials" - func: "run aws ECS auth test" + - func: "run aws assume role with web identity test" - name: "test-6.0-standalone" tags: ["6.0", "standalone"] @@ -1143,6 +1190,7 @@ tasks: - func: "run aws auth test with aws credentials and session token as environment variables" - func: "run aws auth test with aws EC2 credentials" - func: "run aws ECS auth test" + - func: "run aws assume role with web identity test" - name: "test-7.0-standalone" tags: ["7.0", "standalone"] @@ -1203,6 +1251,7 @@ tasks: - func: "run aws auth test with aws credentials and session token as environment variables" - func: "run aws auth test with aws EC2 credentials" - func: "run aws ECS auth test" + - func: "run aws assume role with web identity test" - name: "test-rapid-standalone" tags: ["rapid", "standalone"] @@ -1263,6 +1312,7 @@ tasks: - func: "run aws auth test with aws credentials and session token as environment variables" - func: "run aws auth test with aws EC2 credentials" - func: "run aws ECS auth test" + - func: "run aws assume role with web identity test" - name: "test-latest-standalone" tags: ["latest", "standalone"] @@ -1324,6 +1374,7 @@ tasks: - func: "run aws auth test with aws credentials and session token as environment variables" - func: "run aws auth test with aws EC2 credentials" - func: "run aws ECS auth test" + - func: "run aws assume role with web identity test" - name: "test-connection-string" commands: diff --git a/src/client/auth/aws.rs b/src/client/auth/aws.rs index 6061a3104..c4730bf6a 100644 --- a/src/client/auth/aws.rs +++ b/src/client/auth/aws.rs @@ -1,10 +1,13 @@ +use std::{fs::File, io::Read}; + use chrono::{offset::Utc, DateTime}; use hmac::Hmac; +use rand::distributions::{Alphanumeric, DistString}; use serde::Deserialize; use sha2::{Digest, Sha256}; use crate::{ - bson::{doc, spec::BinarySubtype, Binary, Bson, Document}, + bson::{doc, rawdoc, spec::BinarySubtype, Binary, Bson, Document}, client::{ auth::{ self, @@ -22,6 +25,7 @@ use crate::{ const AWS_ECS_IP: &str = "169.254.170.2"; const AWS_EC2_IP: &str = "169.254.169.254"; const AWS_LONG_DATE_FMT: &str = "%Y%m%dT%H%M%SZ"; +const MECH_NAME: &str = "MONGODB-AWS"; /// Performs MONGODB-AWS authentication for a given stream. pub(super) async fn authenticate_stream( @@ -34,7 +38,7 @@ pub(super) async fn authenticate_stream( Some("$external") | None => "$external", Some(..) => { return Err(Error::authentication_error( - "MONGODB-AWS", + MECH_NAME, "auth source must be $external", )) } @@ -61,8 +65,7 @@ pub(super) async fn authenticate_stream( let server_first_response = conn.send_command(client_first, None).await?; - let server_first = - ServerFirst::parse(server_first_response.auth_response_body("MONGODB-AWS")?)?; + let server_first = ServerFirst::parse(server_first_response.auth_response_body(MECH_NAME)?)?; server_first.validate(&nonce)?; let aws_credential = AwsCredential::get(credential, http_client).await?; @@ -98,16 +101,16 @@ pub(super) async fn authenticate_stream( let server_second_response = conn.send_command(client_second, None).await?; let server_second = SaslResponse::parse( - "MONGODB-AWS", - server_second_response.auth_response_body("MONGODB-AWS")?, + MECH_NAME, + server_second_response.auth_response_body(MECH_NAME)?, )?; if server_second.conversation_id != server_first.conversation_id { - return Err(Error::invalid_authentication_response("MONGODB-AWS")); + return Err(Error::invalid_authentication_response(MECH_NAME)); } if !server_second.done { - return Err(Error::invalid_authentication_response("MONGODB-AWS")); + return Err(Error::invalid_authentication_response(MECH_NAME)); } Ok(()) @@ -115,14 +118,13 @@ pub(super) async fn authenticate_stream( /// Contains the credentials for MONGODB-AWS authentication. #[derive(Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] pub(crate) struct AwsCredential { - #[serde(rename = "AccessKeyId")] - access_key: String, + access_key_id: String, - #[serde(rename = "SecretAccessKey")] - secret_key: String, + secret_access_key: String, - #[serde(rename = "Token")] + #[serde(alias = "Token")] session_token: Option, } @@ -152,15 +154,15 @@ impl AwsCredential { // found. if let (Some(access_key), Some(secret_key)) = (access_key, secret_key) { return Ok(Self { - access_key, - secret_key, + access_key_id: access_key, + secret_access_key: secret_key, session_token, }); } if found_access_key || found_secret_key { return Err(Error::authentication_error( - "MONGODB-AWS", + MECH_NAME, "cannot specify only one of access key and secret key; either both or neither \ must be provided", )); @@ -168,11 +170,23 @@ impl AwsCredential { if session_token.is_some() { return Err(Error::authentication_error( - "MONGODB-AWS", + MECH_NAME, "cannot specify session token without both access key and secret key", )); } + if let (Ok(token_file), Ok(role_arn)) = ( + std::env::var("AWS_WEB_IDENTITY_TOKEN_FILE"), + std::env::var("AWS_ROLE_ARN"), + ) { + return Self::get_from_assume_role_with_web_identity_request( + token_file, + role_arn, + http_client, + ) + .await; + } + if let Ok(relative_uri) = std::env::var("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") { Self::get_from_ecs(relative_uri, http_client).await } else { @@ -180,26 +194,71 @@ impl AwsCredential { } } + async fn get_from_assume_role_with_web_identity_request( + token_file: String, + role_arn: String, + http_client: &HttpClient, + ) -> Result { + let mut file = File::open(&token_file).map_err(|_| { + Error::authentication_error(MECH_NAME, "could not open identity token file") + })?; + let mut buffer = Vec::::new(); + file.read_to_end(&mut buffer).map_err(|_| { + Error::authentication_error(MECH_NAME, "could not read identity token file") + })?; + let token = std::str::from_utf8(&buffer).map_err(|_| { + Error::authentication_error(MECH_NAME, "could not read identity token file") + })?; + + let session_name = std::env::var("AWS_ROLE_SESSION_NAME") + .unwrap_or_else(|_| Alphanumeric.sample_string(&mut rand::thread_rng(), 10)); + + let query = rawdoc! { + "Action": "AssumeRoleWithWebIdentity", + "RoleSessionName": session_name, + "RoleArn": role_arn, + "WebIdentityToken": token, + "Version": "2011-06-15", + }; + + let response = http_client + .get("https://sts.amazonaws.com/") + .headers(&[("Accept", "application/json")]) + .query(query) + .send::() + .await + .map_err(|_| Error::unknown_authentication_error(MECH_NAME))?; + + let credential = response + .get_document("AssumeRoleWithWebIdentityResponse") + .and_then(|d| d.get_document("AssumeRoleWithWebIdentityResult")) + .and_then(|d| d.get_document("Credentials")) + .map_err(|_| Error::unknown_authentication_error(MECH_NAME))? + .to_owned(); + + Ok(bson::from_document(credential)?) + } + /// Obtains credentials from the ECS endpoint. async fn get_from_ecs(relative_uri: String, http_client: &HttpClient) -> Result { // Use the local IP address that AWS uses for ECS agents. let uri = format!("http://{}/{}", AWS_ECS_IP, relative_uri); http_client - .get_and_deserialize_json(&uri, None) + .get(&uri) + .send() .await - .map_err(|_| Error::unknown_authentication_error("MONGODB-AWS")) + .map_err(|_| Error::unknown_authentication_error(MECH_NAME)) } /// Obtains temporary credentials for an EC2 instance to use for authentication. async fn get_from_ec2(http_client: &HttpClient) -> Result { let temporary_token = http_client - .put_and_read_string( - &format!("http://{}/latest/api/token", AWS_EC2_IP), - &[("X-aws-ec2-metadata-token-ttl-seconds", "30")], - ) + .put(&format!("http://{}/latest/api/token", AWS_EC2_IP)) + .headers(&[("X-aws-ec2-metadata-token-ttl-seconds", "30")]) + .send_and_get_string() .await - .map_err(|_| Error::unknown_authentication_error("MONGODB-AWS"))?; + .map_err(|_| Error::unknown_authentication_error(MECH_NAME))?; let role_name_uri = format!( "http://{}/latest/meta-data/iam/security-credentials/", @@ -207,22 +266,20 @@ impl AwsCredential { ); let role_name = http_client - .get_and_read_string( - &role_name_uri, - &[("X-aws-ec2-metadata-token", &temporary_token[..])], - ) + .get(&role_name_uri) + .headers(&[("X-aws-ec2-metadata-token", &temporary_token[..])]) + .send_and_get_string() .await - .map_err(|_| Error::unknown_authentication_error("MONGODB-AWS"))?; + .map_err(|_| Error::unknown_authentication_error(MECH_NAME))?; let credential_uri = format!("{}/{}", role_name_uri, role_name); http_client - .get_and_deserialize_json( - &credential_uri, - &[("X-aws-ec2-metadata-token", &temporary_token[..])], - ) + .get(&credential_uri) + .headers(&[("X-aws-ec2-metadata-token", &temporary_token[..])]) + .send() .await - .map_err(|_| Error::unknown_authentication_error("MONGODB-AWS")) + .map_err(|_| Error::unknown_authentication_error(MECH_NAME)) } /// Computes the signed authorization header for the credentials to send to the server in a sasl @@ -319,16 +376,15 @@ impl AwsCredential { hashed_request = hashed_request, ); - let first_hmac_key = format!("AWS4{}", self.secret_key); + let first_hmac_key = format!("AWS4{}", self.secret_access_key); let k_date = - auth::mac::>(first_hmac_key.as_ref(), small_date.as_ref(), "MONGODB-AWS")?; - let k_region = auth::mac::>(k_date.as_ref(), region.as_ref(), "MONGODB-AWS")?; - let k_service = auth::mac::>(k_region.as_ref(), b"sts", "MONGODB-AWS")?; - let k_signing = - auth::mac::>(k_service.as_ref(), b"aws4_request", "MONGODB-AWS")?; + auth::mac::>(first_hmac_key.as_ref(), small_date.as_ref(), MECH_NAME)?; + let k_region = auth::mac::>(k_date.as_ref(), region.as_ref(), MECH_NAME)?; + let k_service = auth::mac::>(k_region.as_ref(), b"sts", MECH_NAME)?; + let k_signing = auth::mac::>(k_service.as_ref(), b"aws4_request", MECH_NAME)?; let signature_bytes = - auth::mac::>(k_signing.as_ref(), string_to_sign.as_ref(), "MONGODB-AWS")?; + auth::mac::>(k_signing.as_ref(), string_to_sign.as_ref(), MECH_NAME)?; let signature = hex::encode(signature_bytes); #[rustfmt::skip] @@ -339,7 +395,7 @@ impl AwsCredential { SignedHeaders={signed_headers}, \ Signature={signature}\ ", - access_key = self.access_key, + access_key = self.access_key_id, small_date = small_date, region = region, signed_headers = signed_headers, @@ -351,12 +407,12 @@ impl AwsCredential { #[cfg(feature = "in-use-encryption-unstable")] pub(crate) fn access_key(&self) -> &str { - &self.access_key + &self.access_key_id } #[cfg(feature = "in-use-encryption-unstable")] pub(crate) fn secret_key(&self) -> &str { - &self.secret_key + &self.secret_access_key } #[cfg(feature = "in-use-encryption-unstable")] @@ -390,13 +446,13 @@ impl ServerFirst { conversation_id, payload, done, - } = SaslResponse::parse("MONGODB-AWS", response)?; + } = SaslResponse::parse(MECH_NAME, response)?; let ServerFirstPayload { server_nonce, sts_host, } = bson::from_slice(payload.as_slice()) - .map_err(|_| Error::invalid_authentication_response("MONGODB-AWS"))?; + .map_err(|_| Error::invalid_authentication_response(MECH_NAME))?; Ok(Self { conversation_id, @@ -410,32 +466,29 @@ impl ServerFirst { fn validate(&self, nonce: &[u8]) -> Result<()> { if self.done { Err(Error::authentication_error( - "MONGODB-AWS", + MECH_NAME, "handshake terminated early", )) } else if !self.server_nonce.starts_with(nonce) { - Err(Error::authentication_error( - "MONGODB-AWS", - "mismatched nonce", - )) + Err(Error::authentication_error(MECH_NAME, "mismatched nonce")) } else if self.server_nonce.len() != 64 { Err(Error::authentication_error( - "MONGODB-AWS", + MECH_NAME, "incorrect length server nonce", )) } else if self.sts_host.is_empty() { Err(Error::authentication_error( - "MONGODB-AWS", + MECH_NAME, "sts host must be non-empty", )) } else if self.sts_host.as_bytes().len() > 255 { Err(Error::authentication_error( - "MONGODB-AWS", + MECH_NAME, "sts host cannot be more than 255 bytes", )) } else if self.sts_host.split('.').any(|s| s.is_empty()) { Err(Error::authentication_error( - "MONGODB-AWS", + MECH_NAME, "sts host cannot contain empty labels", )) } else { diff --git a/src/client/csfle/state_machine.rs b/src/client/csfle/state_machine.rs index 0d6e9023f..a47c260e7 100644 --- a/src/client/csfle/state_machine.rs +++ b/src/client/csfle/state_machine.rs @@ -265,7 +265,6 @@ impl CryptExecutor { #[cfg(feature = "gcp-kms")] { use crate::runtime::HttpClient; - use reqwest::Method; use serde::Deserialize; #[derive(Deserialize)] @@ -293,26 +292,15 @@ impl CryptExecutor { "http://{}/computeMetadata/v1/instance/service-accounts/default/token", host ); - let headers = vec![("Metadata-Flavor", "Google")]; - let response = http_client - .request(Method::GET, &uri, &headers) + + let response: ResponseBody = http_client + .get(&uri) + .headers(&[("Metadata-Flavor", "Google")]) + .send() .await .map_err(|e| kms_error(e.to_string()))?; - - if response.status().as_u16() != 200 { - let error = match response.text().await { - Ok(text) => text, - Err(e) => format!("could not parse HTTP response: {}", e), - }; - return Err(kms_error(error)); - } - - let body: ResponseBody = response.json().await.map_err(|e| { - let error = format!("could not parse HTTP response: {}", e); - kms_error(error) - })?; kms_providers - .append("gcp", rawdoc! { "accessToken": body.access_token }); + .append("gcp", rawdoc! { "accessToken": response.access_token }); } #[cfg(not(feature = "gcp-kms"))] { @@ -482,7 +470,9 @@ pub(crate) mod azure { let now = Instant::now(); let server_response: ServerResponse = self .http - .get_and_deserialize_json(self.make_url()?, &self.make_headers()) + .get(self.make_url()?) + .headers(&self.make_headers()) + .send() .await .map_err(|e| Error::authentication_error("azure imds", &format!("{}", e)))?; let expires_in_secs: u64 = server_response.expires_in.parse().map_err(|e| { diff --git a/src/runtime/http.rs b/src/runtime/http.rs index 2a773424c..93e8bf214 100644 --- a/src/runtime/http.rs +++ b/src/runtime/http.rs @@ -2,7 +2,7 @@ #![allow(unused)] use reqwest::{IntoUrl, Method, Response}; -use serde::Deserialize; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use crate::error::{Error, Result}; @@ -11,80 +11,64 @@ pub(crate) struct HttpClient { inner: reqwest::Client, } -impl HttpClient { - pub(crate) fn with_timeout(timeout: std::time::Duration) -> Result { - let inner = reqwest::Client::builder() - .timeout(timeout) - .build() - .map_err(|e| Error::internal(format!("error initializing http client: {}", e)))?; - Ok(Self { inner }) +pub(crate) struct HttpRequest { + inner: reqwest::RequestBuilder, +} + +impl From for HttpRequest { + fn from(value: reqwest::RequestBuilder) -> Self { + Self { inner: value } } +} - /// Executes an HTTP GET request and deserializes the JSON response. - pub(crate) async fn get_and_deserialize_json<'a, T>( - &self, - uri: impl IntoUrl, +impl HttpRequest { + /// Sets the headers for the request. + pub(crate) fn headers<'a>( + self, headers: impl IntoIterator, - ) -> reqwest::Result - where - T: for<'de> Deserialize<'de>, - { - let value = self - .request(Method::GET, uri, headers) - .await? - .json() - .await?; - - Ok(value) + ) -> Self { + headers + .into_iter() + .fold(self.inner, |request, (k, v)| request.header(*k, *v)) + .into() } - /// Executes an HTTP GET request and returns the response body as a string. - pub(crate) async fn get_and_read_string<'a>( - &self, - uri: &str, - headers: impl IntoIterator, - ) -> reqwest::Result { - self.request_and_read_string(Method::GET, uri, headers) - .await + /// Sets the query for the request. The query can be any value with key-value pairs that + /// implements Serialize. + pub(crate) fn query(self, query: impl Serialize) -> Self { + self.inner.query(&query).into() } - /// Executes an HTTP PUT request and returns the response body as a string. - pub(crate) async fn put_and_read_string<'a>( - &self, - uri: &str, - headers: impl IntoIterator, - ) -> reqwest::Result { - self.request_and_read_string(Method::PUT, uri, headers) - .await + /// Sends the request via the HttpClient it was created from and returns the result as the given + /// type T. + pub(crate) async fn send(self) -> reqwest::Result { + self.inner.send().await?.json().await } - /// Executes an HTTP request and returns the response body as a string. - pub(crate) async fn request_and_read_string<'a>( - &self, - method: Method, - uri: &str, - headers: impl IntoIterator, - ) -> reqwest::Result { - let text = self.request(method, uri, headers).await?.text().await?; + /// Sends the request via the HttpClient it was created from and returns the result as a string. + pub(crate) async fn send_and_get_string(self) -> reqwest::Result { + self.inner.send().await?.text().await + } +} - Ok(text) +impl HttpClient { + pub(crate) fn with_timeout(timeout: std::time::Duration) -> Result { + let inner = reqwest::Client::builder() + .timeout(timeout) + .build() + .map_err(|e| Error::internal(format!("error initializing http client: {}", e)))?; + Ok(Self { inner }) } - /// Executes an HTTP request and returns the response. - pub(crate) async fn request<'a>( - &self, - method: Method, - uri: impl IntoUrl, - headers: impl IntoIterator, - ) -> reqwest::Result { - let response = headers - .into_iter() - .fold(self.inner.request(method, uri), |request, (k, v)| { - request.header(*k, *v) - }) - .send() - .await?; + /// Creates an HTTP get request. One of the send methods defined on HttpRequest must be called + /// for the request to be executed. + pub(crate) fn get(&self, uri: impl IntoUrl) -> HttpRequest { + self.inner.request(Method::GET, uri).into() + } - Ok(response) + /// Creates an HTTP put request. One of the send methods defined on HttpRequest must be called + /// for the request to be executed. + pub(crate) fn put(&self, uri: impl IntoUrl) -> HttpRequest { + self.inner.request(Method::PUT, uri).into() } }