diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs index 786ccd20f18a..4b633d9f5d24 100644 --- a/object_store/src/aws/mod.rs +++ b/object_store/src/aws/mod.rs @@ -37,9 +37,11 @@ use chrono::{DateTime, Utc}; use futures::stream::BoxStream; use futures::TryStreamExt; use itertools::Itertools; +use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; use std::collections::BTreeSet; use std::ops::Range; +use std::str::FromStr; use std::sync::Arc; use tokio::io::AsyncWrite; use tracing::info; @@ -51,6 +53,7 @@ use crate::aws::credential::{ StaticCredentialProvider, WebIdentityProvider, }; use crate::multipart::{CloudMultiPartUpload, CloudMultiPartUploadImpl, UploadPart}; +use crate::util::str_is_truthy; use crate::{ ClientOptions, GetResult, ListResult, MultipartId, ObjectMeta, ObjectStore, Path, Result, RetryConfig, StreamExt, @@ -133,13 +136,21 @@ enum Error { #[snafu(display("URL did not match any known pattern for scheme: {}", url))] UrlNotRecognised { url: String }, + + #[snafu(display("Configuration key: '{}' is not known.", key))] + UnknownConfigurationKey { key: String }, } impl From for super::Error { - fn from(err: Error) -> Self { - Self::Generic { - store: "S3", - source: Box::new(err), + fn from(source: Error) -> Self { + match source { + Error::UnknownConfigurationKey { key } => { + Self::UnknownConfigurationKey { store: "S3", key } + } + _ => Self::Generic { + store: "S3", + source: Box::new(source), + }, } } } @@ -379,6 +390,184 @@ pub struct AmazonS3Builder { client_options: ClientOptions, } +/// Configuration keys for [`AmazonS3Builder`] +/// +/// Configuration via keys can be dome via the [`try_with_option`](AmazonS3Builder::try_with_option) +/// or [`with_options`](AmazonS3Builder::try_with_options) methods on the builder. +/// +/// # Example +/// ``` +/// use std::collections::HashMap; +/// use object_store::aws::{AmazonS3Builder, AmazonS3ConfigKey}; +/// +/// let options = HashMap::from([ +/// ("aws_access_key_id", "my-access-key-id"), +/// ("aws_secret_access_key", "my-secret-access-key"), +/// ]); +/// let typed_options = vec![ +/// (AmazonS3ConfigKey::DefaultRegion, "my-default-region"), +/// ]; +/// let azure = AmazonS3Builder::new() +/// .try_with_options(options) +/// .unwrap() +/// .try_with_options(typed_options) +/// .unwrap() +/// .try_with_option(AmazonS3ConfigKey::Region, "my-region") +/// .unwrap(); +/// ``` +#[derive(PartialEq, Eq, Hash, Clone, Debug, Copy, Serialize, Deserialize)] +pub enum AmazonS3ConfigKey { + /// AWS Access Key + /// + /// See [`AmazonS3Builder::with_access_key_id`] for details. + /// + /// Supported keys: + /// - `aws_access_key_id` + /// - `access_key_id` + AccessKeyId, + + /// Secret Access Key + /// + /// See [`AmazonS3Builder::with_secret_access_key`] for details. + /// + /// Supported keys: + /// - `aws_secret_access_key` + /// - `secret_access_key` + SecretAccessKey, + + /// Region + /// + /// See [`AmazonS3Builder::with_region`] for details. + /// + /// Supported keys: + /// - `aws_region` + /// - `region` + Region, + + /// Default region + /// + /// See [`AmazonS3Builder::with_region`] for details. + /// + /// Supported keys: + /// - `aws_default_region` + /// - `default_region` + DefaultRegion, + + /// Bucket name + /// + /// See [`AmazonS3Builder::with_bucket_name`] for details. + /// + /// Supported keys: + /// - `aws_bucket` + /// - `aws_bucket_name` + /// - `bucket` + /// - `bucket_name` + Bucket, + + /// Sets custom endpoint for communicating with AWS S3. + /// + /// See [`AmazonS3Builder::with_endpoint`] for details. + /// + /// Supported keys: + /// - `aws_endpoint` + /// - `aws_endpoint_url` + /// - `endpoint` + /// - `endpoint_url` + Endpoint, + + /// Token to use for requests (passed to underlying provider) + /// + /// See [`AmazonS3Builder::with_token`] for details. + /// + /// Supported keys: + /// - `aws_session_token` + /// - `aws_token` + /// - `session_token` + /// - `token` + Token, + + /// Fall back to ImdsV1 + /// + /// See [`AmazonS3Builder::with_imdsv1_fallback`] for details. + /// + /// Supported keys: + /// - `aws_imdsv1_fallback` + /// - `imdsv1_fallback` + ImdsV1Fallback, + + /// If virtual hosted style request has to be used + /// + /// See [`AmazonS3Builder::with_virtual_hosted_style_request`] for details. + /// + /// Supported keys: + /// - `aws_virtual_hosted_style_request` + /// - `virtual_hosted_style_request` + VirtualHostedStyleRequest, + + /// Set the instance metadata endpoint + /// + /// See [`AmazonS3Builder::with_metadata_endpoint`] for details. + /// + /// Supported keys: + /// - `aws_metadata_endpoint` + /// - `metadata_endpoint` + MetadataEndpoint, + + /// AWS profile name + /// + /// Supported keys: + /// - `aws_profile` + /// - `profile` + Profile, +} + +impl AsRef for AmazonS3ConfigKey { + fn as_ref(&self) -> &str { + match self { + Self::AccessKeyId => "aws_access_key_id", + Self::SecretAccessKey => "aws_secret_access_key", + Self::Region => "aws_region", + Self::Bucket => "aws_bucket", + Self::Endpoint => "aws_endpoint", + Self::Token => "aws_session_token", + Self::ImdsV1Fallback => "aws_imdsv1_fallback", + Self::VirtualHostedStyleRequest => "aws_virtual_hosted_style_request", + Self::DefaultRegion => "aws_default_region", + Self::MetadataEndpoint => "aws_metadata_endpoint", + Self::Profile => "aws_profile", + } + } +} + +impl FromStr for AmazonS3ConfigKey { + type Err = super::Error; + + fn from_str(s: &str) -> Result { + match s { + "aws_access_key_id" | "access_key_id" => Ok(Self::AccessKeyId), + "aws_secret_access_key" | "secret_access_key" => Ok(Self::SecretAccessKey), + "aws_default_region" | "default_region" => Ok(Self::DefaultRegion), + "aws_region" | "region" => Ok(Self::Region), + "aws_bucket" | "aws_bucket_name" | "bucket_name" | "bucket" => { + Ok(Self::Bucket) + } + "aws_endpoint_url" | "aws_endpoint" | "endpoint_url" | "endpoint" => { + Ok(Self::Endpoint) + } + "aws_session_token" | "aws_token" | "session_token" | "token" => { + Ok(Self::Token) + } + "aws_virtual_hosted_style_request" | "virtual_hosted_style_request" => { + Ok(Self::VirtualHostedStyleRequest) + } + "aws_profile" | "profile" => Ok(Self::Profile), + "aws_imdsv1_fallback" | "imdsv1_fallback" => Ok(Self::ImdsV1Fallback), + "aws_metadata_endpoint" | "metadata_endpoint" => Ok(Self::MetadataEndpoint), + _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()), + } + } +} + impl AmazonS3Builder { /// Create a new [`AmazonS3Builder`] with default values. pub fn new() -> Self { @@ -407,28 +596,16 @@ impl AmazonS3Builder { pub fn from_env() -> Self { let mut builder: Self = Default::default(); - if let Ok(access_key_id) = std::env::var("AWS_ACCESS_KEY_ID") { - builder.access_key_id = Some(access_key_id); - } - - if let Ok(secret_access_key) = std::env::var("AWS_SECRET_ACCESS_KEY") { - builder.secret_access_key = Some(secret_access_key); - } - - if let Ok(secret) = std::env::var("AWS_DEFAULT_REGION") { - builder.region = Some(secret); - } - - if let Ok(endpoint) = std::env::var("AWS_ENDPOINT") { - builder.endpoint = Some(endpoint); - } - - if let Ok(token) = std::env::var("AWS_SESSION_TOKEN") { - builder.token = Some(token); - } - - if let Ok(profile) = std::env::var("AWS_PROFILE") { - builder.profile = Some(profile); + for (os_key, os_value) in std::env::vars_os() { + if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) { + if key.starts_with("AWS_") { + if let Ok(config_key) = + AmazonS3ConfigKey::from_str(&key.to_ascii_lowercase()) + { + builder = builder.try_with_option(config_key, value).unwrap(); + } + } + } } // This env var is set in ECS @@ -442,7 +619,7 @@ impl AmazonS3Builder { if let Ok(text) = std::env::var("AWS_ALLOW_HTTP") { builder.client_options = - builder.client_options.with_allow_http(text == "true"); + builder.client_options.with_allow_http(str_is_truthy(&text)); } builder @@ -472,6 +649,55 @@ impl AmazonS3Builder { self } + /// Set an option on the builder via a key - value pair. + /// + /// This method will return an `UnknownConfigKey` error if key cannot be parsed into [`AmazonS3ConfigKey`]. + pub fn try_with_option( + mut self, + key: impl AsRef, + value: impl Into, + ) -> Result { + match AmazonS3ConfigKey::from_str(key.as_ref())? { + AmazonS3ConfigKey::AccessKeyId => self.access_key_id = Some(value.into()), + AmazonS3ConfigKey::SecretAccessKey => { + self.secret_access_key = Some(value.into()) + } + AmazonS3ConfigKey::Region => self.region = Some(value.into()), + AmazonS3ConfigKey::Bucket => self.bucket_name = Some(value.into()), + AmazonS3ConfigKey::Endpoint => self.endpoint = Some(value.into()), + AmazonS3ConfigKey::Token => self.token = Some(value.into()), + AmazonS3ConfigKey::ImdsV1Fallback => { + self.imdsv1_fallback = str_is_truthy(&value.into()) + } + AmazonS3ConfigKey::VirtualHostedStyleRequest => { + self.virtual_hosted_style_request = str_is_truthy(&value.into()) + } + AmazonS3ConfigKey::DefaultRegion => { + self.region = self.region.or_else(|| Some(value.into())) + } + AmazonS3ConfigKey::MetadataEndpoint => { + self.metadata_endpoint = Some(value.into()) + } + AmazonS3ConfigKey::Profile => self.profile = Some(value.into()), + }; + Ok(self) + } + + /// Hydrate builder from key value pairs + /// + /// This method will return an `UnknownConfigKey` error if any key cannot be parsed into [`AmazonS3ConfigKey`]. + pub fn try_with_options< + I: IntoIterator, impl Into)>, + >( + mut self, + options: I, + ) -> Result { + for (key, value) in options { + self = self.try_with_option(key, value)?; + } + Ok(self) + } + /// Sets properties on this builder based on a URL /// /// This is a separate member function to allow fallible computation to @@ -773,6 +999,7 @@ mod tests { put_get_delete_list_opts, rename_and_copy, stream_get, }; use bytes::Bytes; + use std::collections::HashMap; use std::env; const NON_EXISTENT_NAME: &str = "nonexistentname"; @@ -915,6 +1142,73 @@ mod tests { assert_eq!(builder.metadata_endpoint.unwrap(), metadata_uri); } + #[test] + fn s3_test_config_from_map() { + let aws_access_key_id = "object_store:fake_access_key_id".to_string(); + let aws_secret_access_key = "object_store:fake_secret_key".to_string(); + let aws_default_region = "object_store:fake_default_region".to_string(); + let aws_endpoint = "object_store:fake_endpoint".to_string(); + let aws_session_token = "object_store:fake_session_token".to_string(); + let options = HashMap::from([ + ("aws_access_key_id", aws_access_key_id.clone()), + ("aws_secret_access_key", aws_secret_access_key), + ("aws_default_region", aws_default_region.clone()), + ("aws_endpoint", aws_endpoint.clone()), + ("aws_session_token", aws_session_token.clone()), + ]); + + let builder = AmazonS3Builder::new() + .try_with_options(&options) + .unwrap() + .try_with_option("aws_secret_access_key", "new-secret-key") + .unwrap(); + assert_eq!(builder.access_key_id.unwrap(), aws_access_key_id.as_str()); + assert_eq!(builder.secret_access_key.unwrap(), "new-secret-key"); + assert_eq!(builder.region.unwrap(), aws_default_region); + assert_eq!(builder.endpoint.unwrap(), aws_endpoint); + assert_eq!(builder.token.unwrap(), aws_session_token); + } + + #[test] + fn s3_test_config_from_typed_map() { + let aws_access_key_id = "object_store:fake_access_key_id".to_string(); + let aws_secret_access_key = "object_store:fake_secret_key".to_string(); + let aws_default_region = "object_store:fake_default_region".to_string(); + let aws_endpoint = "object_store:fake_endpoint".to_string(); + let aws_session_token = "object_store:fake_session_token".to_string(); + let options = HashMap::from([ + (AmazonS3ConfigKey::AccessKeyId, aws_access_key_id.clone()), + (AmazonS3ConfigKey::SecretAccessKey, aws_secret_access_key), + (AmazonS3ConfigKey::DefaultRegion, aws_default_region.clone()), + (AmazonS3ConfigKey::Endpoint, aws_endpoint.clone()), + (AmazonS3ConfigKey::Token, aws_session_token.clone()), + ]); + + let builder = AmazonS3Builder::new() + .try_with_options(&options) + .unwrap() + .try_with_option(AmazonS3ConfigKey::SecretAccessKey, "new-secret-key") + .unwrap(); + assert_eq!(builder.access_key_id.unwrap(), aws_access_key_id.as_str()); + assert_eq!(builder.secret_access_key.unwrap(), "new-secret-key"); + assert_eq!(builder.region.unwrap(), aws_default_region); + assert_eq!(builder.endpoint.unwrap(), aws_endpoint); + assert_eq!(builder.token.unwrap(), aws_session_token); + } + + #[test] + fn s3_test_config_fallible_options() { + let aws_access_key_id = "object_store:fake_access_key_id".to_string(); + let aws_secret_access_key = "object_store:fake_secret_key".to_string(); + let options = HashMap::from([ + ("aws_access_key_id", aws_access_key_id), + ("invalid-key", aws_secret_access_key), + ]); + + let builder = AmazonS3Builder::new().try_with_options(&options); + assert!(builder.is_err()); + } + #[tokio::test] async fn s3_test() { let config = maybe_skip_integration!(); diff --git a/object_store/src/azure/mod.rs b/object_store/src/azure/mod.rs index 7cf369de3b3a..416883ac95a2 100644 --- a/object_store/src/azure/mod.rs +++ b/object_store/src/azure/mod.rs @@ -37,16 +37,18 @@ use async_trait::async_trait; use bytes::Bytes; use chrono::{TimeZone, Utc}; use futures::{stream::BoxStream, StreamExt, TryStreamExt}; +use percent_encoding::percent_decode_str; +use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; -use std::collections::BTreeSet; use std::fmt::{Debug, Formatter}; use std::io; use std::ops::Range; use std::sync::Arc; +use std::{collections::BTreeSet, str::FromStr}; use tokio::io::AsyncWrite; use url::Url; -use crate::util::RFC1123_FMT; +use crate::util::{str_is_truthy, RFC1123_FMT}; pub use credential::authority_hosts; mod client; @@ -124,13 +126,28 @@ enum Error { #[snafu(display("URL did not match any known pattern for scheme: {}", url))] UrlNotRecognised { url: String }, + + #[snafu(display("Failed parsing an SAS key"))] + DecodeSasKey { source: std::str::Utf8Error }, + + #[snafu(display("Missing component in SAS query pair"))] + MissingSasComponent {}, + + #[snafu(display("Configuration key: '{}' is not known.", key))] + UnknownConfigurationKey { key: String }, } impl From for super::Error { fn from(source: Error) -> Self { - Self::Generic { - store: "MicrosoftAzure", - source: Box::new(source), + match source { + Error::UnknownConfigurationKey { key } => Self::UnknownConfigurationKey { + store: "MicrosoftAzure", + key, + }, + _ => Self::Generic { + store: "MicrosoftAzure", + source: Box::new(source), + }, } } } @@ -367,6 +384,7 @@ pub struct MicrosoftAzureBuilder { client_secret: Option, tenant_id: Option, sas_query_pairs: Option>, + sas_key: Option, authority_host: Option, url: Option, use_emulator: bool, @@ -374,6 +392,157 @@ pub struct MicrosoftAzureBuilder { client_options: ClientOptions, } +/// Configuration keys for [`MicrosoftAzureBuilder`] +/// +/// Configuration via keys can be dome via the [`try_with_option`](MicrosoftAzureBuilder::try_with_option) +/// or [`with_options`](MicrosoftAzureBuilder::try_with_options) methods on the builder. +/// +/// # Example +/// ``` +/// use std::collections::HashMap; +/// use object_store::azure::{MicrosoftAzureBuilder, AzureConfigKey}; +/// +/// let options = HashMap::from([ +/// ("azure_client_id", "my-client-id"), +/// ("azure_client_secret", "my-account-name"), +/// ]); +/// let typed_options = vec![ +/// (AzureConfigKey::AccountName, "my-account-name"), +/// ]; +/// let azure = MicrosoftAzureBuilder::new() +/// .try_with_options(options) +/// .unwrap() +/// .try_with_options(typed_options) +/// .unwrap() +/// .try_with_option(AzureConfigKey::AuthorityId, "my-tenant-id") +/// .unwrap(); +/// ``` +#[derive(PartialEq, Eq, Hash, Clone, Debug, Copy, Deserialize, Serialize)] +pub enum AzureConfigKey { + /// The name of the azure storage account + /// + /// Supported keys: + /// - `azure_storage_account_name` + /// - `account_name` + AccountName, + + /// Master key for accessing storage account + /// + /// Supported keys: + /// - `azure_storage_account_key` + /// - `azure_storage_access_key` + /// - `azure_storage_master_key` + /// - `access_key` + /// - `account_key` + /// - `master_key` + AccessKey, + + /// Service principal client id for authorizing requests + /// + /// Supported keys: + /// - `azure_storage_client_id` + /// - `azure_client_id` + /// - `client_id` + ClientId, + + /// Service principal client secret for authorizing requests + /// + /// Supported keys: + /// - `azure_storage_client_secret` + /// - `azure_client_secret` + /// - `client_secret` + ClientSecret, + + /// Tenant id used in oauth flows + /// + /// Supported keys: + /// - `azure_storage_tenant_id` + /// - `azure_storage_authority_id` + /// - `azure_tenant_id` + /// - `azure_authority_id` + /// - `tenant_id` + /// - `authority_id` + AuthorityId, + + /// Shared access signature. + /// + /// The signature is expected to be percent-encoded, much like they are provided + /// in the azure storage explorer or azure portal. + /// + /// Supported keys: + /// - `azure_storage_sas_key` + /// - `azure_storage_sas_token` + /// - `sas_key` + /// - `sas_token` + SasKey, + + /// Bearer token + /// + /// Supported keys: + /// - `azure_storage_token` + /// - `bearer_token` + /// - `token` + Token, + + /// Use object store with azurite storage emulator + /// + /// Supported keys: + /// - `azure_storage_use_emulator` + /// - `object_store_use_emulator` + /// - `use_emulator` + UseEmulator, +} + +impl AsRef for AzureConfigKey { + fn as_ref(&self) -> &str { + match self { + Self::AccountName => "azure_storage_account_name", + Self::AccessKey => "azure_storage_account_key", + Self::ClientId => "azure_storage_client_id", + Self::ClientSecret => "azure_storage_client_secret", + Self::AuthorityId => "azure_storage_tenant_id", + Self::SasKey => "azure_storage_sas_key", + Self::Token => "azure_storage_token", + Self::UseEmulator => "azure_storage_use_emulator", + } + } +} + +impl FromStr for AzureConfigKey { + type Err = super::Error; + + fn from_str(s: &str) -> Result { + match s { + "azure_storage_account_key" + | "azure_storage_access_key" + | "azure_storage_master_key" + | "master_key" + | "account_key" + | "access_key" => Ok(Self::AccessKey), + "azure_storage_account_name" | "account_name" => Ok(Self::AccountName), + "azure_storage_client_id" | "azure_client_id" | "client_id" => { + Ok(Self::ClientId) + } + "azure_storage_client_secret" | "azure_client_secret" | "client_secret" => { + Ok(Self::ClientSecret) + } + "azure_storage_tenant_id" + | "azure_storage_authority_id" + | "azure_tenant_id" + | "azure_authority_id" + | "tenant_id" + | "authority_id" => Ok(Self::AuthorityId), + "azure_storage_sas_key" + | "azure_storage_sas_token" + | "sas_key" + | "sas_token" => Ok(Self::SasKey), + "azure_storage_token" | "bearer_token" | "token" => Ok(Self::Token), + "azure_storage_use_emulator" | "use_emulator" => Ok(Self::UseEmulator), + _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()), + } + } +} + impl Debug for MicrosoftAzureBuilder { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( @@ -409,27 +578,21 @@ impl MicrosoftAzureBuilder { /// ``` pub fn from_env() -> Self { let mut builder = Self::default(); - - if let Ok(account_name) = std::env::var("AZURE_STORAGE_ACCOUNT_NAME") { - builder.account_name = Some(account_name); - } - - if let Ok(access_key) = std::env::var("AZURE_STORAGE_ACCOUNT_KEY") { - builder.access_key = Some(access_key); - } else if let Ok(access_key) = std::env::var("AZURE_STORAGE_ACCESS_KEY") { - builder.access_key = Some(access_key); - } - - if let Ok(client_id) = std::env::var("AZURE_STORAGE_CLIENT_ID") { - builder.client_id = Some(client_id); - } - - if let Ok(client_secret) = std::env::var("AZURE_STORAGE_CLIENT_SECRET") { - builder.client_secret = Some(client_secret); + for (os_key, os_value) in std::env::vars_os() { + if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) { + if key.starts_with("AZURE_") { + if let Ok(config_key) = + AzureConfigKey::from_str(&key.to_ascii_lowercase()) + { + builder = builder.try_with_option(config_key, value).unwrap(); + } + } + } } - if let Ok(tenant_id) = std::env::var("AZURE_STORAGE_TENANT_ID") { - builder.tenant_id = Some(tenant_id); + if let Ok(text) = std::env::var("AZURE_ALLOW_HTTP") { + builder.client_options = + builder.client_options.with_allow_http(str_is_truthy(&text)); } builder @@ -462,6 +625,40 @@ impl MicrosoftAzureBuilder { self } + /// Set an option on the builder via a key - value pair. + pub fn try_with_option( + mut self, + key: impl AsRef, + value: impl Into, + ) -> Result { + match AzureConfigKey::from_str(key.as_ref())? { + AzureConfigKey::AccessKey => self.access_key = Some(value.into()), + AzureConfigKey::AccountName => self.account_name = Some(value.into()), + AzureConfigKey::ClientId => self.client_id = Some(value.into()), + AzureConfigKey::ClientSecret => self.client_secret = Some(value.into()), + AzureConfigKey::AuthorityId => self.tenant_id = Some(value.into()), + AzureConfigKey::SasKey => self.sas_key = Some(value.into()), + AzureConfigKey::Token => self.bearer_token = Some(value.into()), + AzureConfigKey::UseEmulator => { + self.use_emulator = str_is_truthy(&value.into()) + } + }; + Ok(self) + } + + /// Hydrate builder from key value pairs + pub fn try_with_options< + I: IntoIterator, impl Into)>, + >( + mut self, + options: I, + ) -> Result { + for (key, value) in options { + self = self.try_with_option(key, value)?; + } + Ok(self) + } + /// Sets properties on this builder based on a URL /// /// This is a separate member function to allow fallible computation to @@ -636,6 +833,8 @@ impl MicrosoftAzureBuilder { )) } else if let Some(query_pairs) = self.sas_query_pairs { Ok(credential::CredentialProvider::SASToken(query_pairs)) + } else if let Some(sas) = self.sas_key { + Ok(credential::CredentialProvider::SASToken(split_sas(&sas)?)) } else { Err(Error::MissingCredentials {}) }?; @@ -673,6 +872,25 @@ fn url_from_env(env_name: &str, default_url: &str) -> Result { Ok(url) } +fn split_sas(sas: &str) -> Result, Error> { + let sas = percent_decode_str(sas) + .decode_utf8() + .context(DecodeSasKeySnafu {})?; + let kv_str_pairs = sas + .trim_start_matches('?') + .split('&') + .filter(|s| !s.chars().all(char::is_whitespace)); + let mut pairs = Vec::new(); + for kv_pair_str in kv_str_pairs { + let (k, v) = kv_pair_str + .trim() + .split_once('=') + .ok_or(Error::MissingSasComponent {})?; + pairs.push((k.into(), v.into())) + } + Ok(pairs) +} + #[cfg(test)] mod tests { use super::*; @@ -680,6 +898,7 @@ mod tests { copy_if_not_exists, list_uses_directories_correctly, list_with_delimiter, put_get_delete_list, put_get_delete_list_opts, rename_and_copy, stream_get, }; + use std::collections::HashMap; use std::env; // Helper macro to skip tests if TEST_INTEGRATION and the Azure environment @@ -832,4 +1051,76 @@ mod tests { builder.parse_url(case).unwrap_err(); } } + + #[test] + fn azure_test_config_from_map() { + let azure_client_id = "object_store:fake_access_key_id"; + let azure_storage_account_name = "object_store:fake_secret_key"; + let azure_storage_token = "object_store:fake_default_region"; + let options = HashMap::from([ + ("azure_client_id", azure_client_id), + ("azure_storage_account_name", azure_storage_account_name), + ("azure_storage_token", azure_storage_token), + ]); + + let builder = MicrosoftAzureBuilder::new() + .try_with_options(options) + .unwrap(); + assert_eq!(builder.client_id.unwrap(), azure_client_id); + assert_eq!(builder.account_name.unwrap(), azure_storage_account_name); + assert_eq!(builder.bearer_token.unwrap(), azure_storage_token); + } + + #[test] + fn azure_test_config_from_typed_map() { + let azure_client_id = "object_store:fake_access_key_id".to_string(); + let azure_storage_account_name = "object_store:fake_secret_key".to_string(); + let azure_storage_token = "object_store:fake_default_region".to_string(); + let options = HashMap::from([ + (AzureConfigKey::ClientId, azure_client_id.clone()), + ( + AzureConfigKey::AccountName, + azure_storage_account_name.clone(), + ), + (AzureConfigKey::Token, azure_storage_token.clone()), + ]); + + let builder = MicrosoftAzureBuilder::new() + .try_with_options(&options) + .unwrap(); + assert_eq!(builder.client_id.unwrap(), azure_client_id); + assert_eq!(builder.account_name.unwrap(), azure_storage_account_name); + assert_eq!(builder.bearer_token.unwrap(), azure_storage_token); + } + + #[test] + fn azure_test_config_fallible_options() { + let azure_client_id = "object_store:fake_access_key_id".to_string(); + let azure_storage_token = "object_store:fake_default_region".to_string(); + let options = HashMap::from([ + ("azure_client_id", azure_client_id), + ("invalid-key", azure_storage_token), + ]); + + let builder = MicrosoftAzureBuilder::new().try_with_options(&options); + assert!(builder.is_err()); + } + + #[test] + fn azure_test_split_sas() { + let raw_sas = "?sv=2021-10-04&st=2023-01-04T17%3A48%3A57Z&se=2023-01-04T18%3A15%3A00Z&sr=c&sp=rcwl&sig=C7%2BZeEOWbrxPA3R0Cw%2Fw1EZz0%2B4KBvQexeKZKe%2BB6h0%3D"; + let expected = vec![ + ("sv".to_string(), "2021-10-04".to_string()), + ("st".to_string(), "2023-01-04T17:48:57Z".to_string()), + ("se".to_string(), "2023-01-04T18:15:00Z".to_string()), + ("sr".to_string(), "c".to_string()), + ("sp".to_string(), "rcwl".to_string()), + ( + "sig".to_string(), + "C7+ZeEOWbrxPA3R0Cw/w1EZz0+4KBvQexeKZKe+B6h0=".to_string(), + ), + ]; + let pairs = split_sas(raw_sas).unwrap(); + assert_eq!(expected, pairs); + } } diff --git a/object_store/src/gcp/mod.rs b/object_store/src/gcp/mod.rs index f2638748f6ca..177812fa8930 100644 --- a/object_store/src/gcp/mod.rs +++ b/object_store/src/gcp/mod.rs @@ -33,6 +33,7 @@ use std::collections::BTreeSet; use std::fs::File; use std::io::{self, BufReader}; use std::ops::Range; +use std::str::FromStr; use std::sync::Arc; use async_trait::async_trait; @@ -42,6 +43,7 @@ use futures::{stream::BoxStream, StreamExt, TryStreamExt}; use percent_encoding::{percent_encode, NON_ALPHANUMERIC}; use reqwest::header::RANGE; use reqwest::{header, Client, Method, Response, StatusCode}; +use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; use tokio::io::AsyncWrite; use url::Url; @@ -145,6 +147,9 @@ enum Error { #[snafu(display("URL did not match any known pattern for scheme: {}", url))] UrlNotRecognised { url: String }, + + #[snafu(display("Configuration key: '{}' is not known.", key))] + UnknownConfigurationKey { key: String }, } impl From for super::Error { @@ -164,6 +169,9 @@ impl From for super::Error { source: Box::new(source), path, }, + Error::UnknownConfigurationKey { key } => { + Self::UnknownConfigurationKey { store: "GCS", key } + } _ => Self::Generic { store: "GCS", source: Box::new(err), @@ -796,6 +804,74 @@ pub struct GoogleCloudStorageBuilder { client_options: ClientOptions, } +/// Configuration keys for [`GoogleCloudStorageBuilder`] +/// +/// Configuration via keys can be dome via the [`try_with_option`](GoogleCloudStorageBuilder::try_with_option) +/// or [`with_options`](GoogleCloudStorageBuilder::try_with_options) methods on the builder. +/// +/// # Example +/// ``` +/// use std::collections::HashMap; +/// use object_store::gcp::{GoogleCloudStorageBuilder, GoogleConfigKey}; +/// +/// let options = HashMap::from([ +/// ("google_service_account", "my-service-account"), +/// ]); +/// let typed_options = vec![ +/// (GoogleConfigKey::Bucket, "my-bucket"), +/// ]; +/// let azure = GoogleCloudStorageBuilder::new() +/// .try_with_options(options) +/// .unwrap() +/// .try_with_options(typed_options) +/// .unwrap() +/// .try_with_option(GoogleConfigKey::Bucket, "my-new-bucket") +/// .unwrap(); +/// ``` +#[derive(PartialEq, Eq, Hash, Clone, Debug, Copy, Serialize, Deserialize)] +pub enum GoogleConfigKey { + /// Path to the service account file + /// + /// Supported keys: + /// - `google_service_account` + /// - `service_account` + ServiceAccount, + + /// Bucket name + /// + /// See [`GoogleCloudStorageBuilder::with_bucket_name`] for details. + /// + /// Supported keys: + /// - `google_bucket` + /// - `google_bucket_name` + /// - `bucket` + /// - `bucket_name` + Bucket, +} + +impl AsRef for GoogleConfigKey { + fn as_ref(&self) -> &str { + match self { + Self::ServiceAccount => "google_service_account", + Self::Bucket => "google_bucket", + } + } +} + +impl FromStr for GoogleConfigKey { + type Err = super::Error; + + fn from_str(s: &str) -> Result { + match s { + "google_service_account" | "service_account" => Ok(Self::ServiceAccount), + "google_bucket" | "google_bucket_name" | "bucket" | "bucket_name" => { + Ok(Self::Bucket) + } + _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()), + } + } +} + impl Default for GoogleCloudStorageBuilder { fn default() -> Self { Self { @@ -835,8 +911,16 @@ impl GoogleCloudStorageBuilder { builder.service_account_path = Some(service_account_path); } - if let Ok(service_account_path) = std::env::var("GOOGLE_SERVICE_ACCOUNT") { - builder.service_account_path = Some(service_account_path); + for (os_key, os_value) in std::env::vars_os() { + if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) { + if key.starts_with("GOOGLE_") { + if let Ok(config_key) = + GoogleConfigKey::from_str(&key.to_ascii_lowercase()) + { + builder = builder.try_with_option(config_key, value).unwrap(); + } + } + } } builder @@ -863,6 +947,34 @@ impl GoogleCloudStorageBuilder { self } + /// Set an option on the builder via a key - value pair. + pub fn try_with_option( + mut self, + key: impl AsRef, + value: impl Into, + ) -> Result { + match GoogleConfigKey::from_str(key.as_ref())? { + GoogleConfigKey::ServiceAccount => { + self.service_account_path = Some(value.into()) + } + GoogleConfigKey::Bucket => self.bucket_name = Some(value.into()), + }; + Ok(self) + } + + /// Hydrate builder from key value pairs + pub fn try_with_options< + I: IntoIterator, impl Into)>, + >( + mut self, + options: I, + ) -> Result { + for (key, value) in options { + self = self.try_with_option(key, value)?; + } + Ok(self) + } + /// Sets properties on this builder based on a URL /// /// This is a separate member function to allow fallible computation to @@ -995,9 +1107,9 @@ fn convert_object_meta(object: &Object) -> Result { #[cfg(test)] mod test { - use std::env; - use bytes::Bytes; + use std::collections::HashMap; + use std::env; use crate::{ tests::{ @@ -1205,4 +1317,58 @@ mod test { builder.parse_url(case).unwrap_err(); } } + + #[test] + fn gcs_test_config_from_map() { + let google_service_account = "object_store:fake_service_account".to_string(); + let google_bucket_name = "object_store:fake_bucket".to_string(); + let options = HashMap::from([ + ("google_service_account", google_service_account.clone()), + ("google_bucket_name", google_bucket_name.clone()), + ]); + + let builder = GoogleCloudStorageBuilder::new() + .try_with_options(&options) + .unwrap(); + assert_eq!( + builder.service_account_path.unwrap(), + google_service_account.as_str() + ); + assert_eq!(builder.bucket_name.unwrap(), google_bucket_name.as_str()); + } + + #[test] + fn gcs_test_config_from_typed_map() { + let google_service_account = "object_store:fake_service_account".to_string(); + let google_bucket_name = "object_store:fake_bucket".to_string(); + let options = HashMap::from([ + ( + GoogleConfigKey::ServiceAccount, + google_service_account.clone(), + ), + (GoogleConfigKey::Bucket, google_bucket_name.clone()), + ]); + + let builder = GoogleCloudStorageBuilder::new() + .try_with_options(&options) + .unwrap(); + assert_eq!( + builder.service_account_path.unwrap(), + google_service_account.as_str() + ); + assert_eq!(builder.bucket_name.unwrap(), google_bucket_name.as_str()); + } + + #[test] + fn gcs_test_config_fallible_options() { + let google_service_account = "object_store:fake_service_account".to_string(); + let google_bucket_name = "object_store:fake_bucket".to_string(); + let options = HashMap::from([ + ("google_service_account", google_service_account), + ("invalid-key", google_bucket_name), + ]); + + let builder = GoogleCloudStorageBuilder::new().try_with_options(&options); + assert!(builder.is_err()); + } } diff --git a/object_store/src/lib.rs b/object_store/src/lib.rs index 425c5cdba1d1..4ec58c387e49 100644 --- a/object_store/src/lib.rs +++ b/object_store/src/lib.rs @@ -555,6 +555,13 @@ pub enum Error { #[snafu(display("Operation not yet implemented."))] NotImplemented, + + #[snafu(display( + "Configuration key: '{}' is not valid for store '{}'.", + key, + store + ))] + UnknownConfigurationKey { store: &'static str, key: String }, } impl From for std::io::Error { diff --git a/object_store/src/util.rs b/object_store/src/util.rs index e592e7b64f2d..08bfd86d9f67 100644 --- a/object_store/src/util.rs +++ b/object_store/src/util.rs @@ -185,6 +185,15 @@ fn merge_ranges( ret } +#[allow(dead_code)] +pub(crate) fn str_is_truthy(val: &str) -> bool { + val.eq_ignore_ascii_case("1") + | val.eq_ignore_ascii_case("true") + | val.eq_ignore_ascii_case("on") + | val.eq_ignore_ascii_case("yes") + | val.eq_ignore_ascii_case("y") +} + #[cfg(test)] mod tests { use super::*;