Merge pull request #5025 from Xuanwo/azblob

feat: Implement azblob support
databendlabs · Apr 24, 2022 · df4ff7d · df4ff7d · vercel · Apr 24, 2022
2 parents 3337389 + 5a7edb6
commit df4ff7d
Show file tree

Hide file tree

Showing 18 changed files with 103 additions and 50 deletions.
diff --git a/docker/databend-query-docker.toml b/docker/databend-query-docker.toml
@@ -55,4 +55,4 @@ data_path = "stateless_test_data"
 [storage.s3]
 
 # Azure storage
-[storage.azure_storage_blob]
+[storage.azblob]
diff --git a/docs/doc/10-deploy/00-local.md b/docs/doc/10-deploy/00-local.md
@@ -161,7 +161,7 @@ data_path = "benddata/datas"
 
 [storage.s3]
 
-[storage.azure_storage_blob]
+[storage.azblob]
 ```
 
 ### 3.2 Start databend-query

diff --git a/docs/doc/10-deploy/01-s3.md b/docs/doc/10-deploy/01-s3.md
@@ -132,7 +132,7 @@ access_key_id = "<your-key-id>"
 // highlight-next-line
 secret_access_key = "<your-access-key>"
 
-[storage.azure_storage_blob]
+[storage.azblob]
 ```
 
 ### 3.2 Start databend-query

diff --git a/docs/doc/10-deploy/02-cos.md b/docs/doc/10-deploy/02-cos.md
@@ -148,7 +148,7 @@ access_key_id = "<your-key-id>"
 // highlight-next-line
 secret_access_key = "<your-access-key>"
 
-[storage.azure_storage_blob]
+[storage.azblob]
 ```
 
 :::tip

diff --git a/docs/doc/10-deploy/03-oss.md b/docs/doc/10-deploy/03-oss.md
@@ -151,7 +151,7 @@ access_key_id = "<your-key-id>"
 // highlight-next-line
 secret_access_key = "<your-access-key>"
 
-[storage.azure_storage_blob]
+[storage.azblob]
 ```
 
 :::tip

diff --git a/docs/doc/10-deploy/05-wasabi.md b/docs/doc/10-deploy/05-wasabi.md
@@ -149,7 +149,7 @@ access_key_id = "<your-key-id>"
 // highlight-next-line
 secret_access_key = "<your-access-key>"
 
-[storage.azure_storage_blob]
+[storage.azblob]
 ```
 
 :::tip

diff --git a/docs/doc/10-deploy/06-scw.md b/docs/doc/10-deploy/06-scw.md
@@ -150,7 +150,7 @@ access_key_id = "<your-access-key>"
 // highlight-next-line
 secret_access_key = "<your-secret-key>"
 
-[storage.azure_storage_blob]
+[storage.azblob]
 ```
 
 :::tip

diff --git a/query/src/configs/config_storage.rs b/query/src/configs/config_storage.rs
@@ -119,26 +119,44 @@ impl fmt::Debug for S3StorageConfig {
 
 #[derive(Clone, PartialEq, Serialize, Deserialize, Default, Args)]
 #[serde(default)]
-pub struct AzureStorageBlobConfig {
-    /// Account for Azure storage
-    #[clap(long = "storage-azblob-account", default_value_t)]
-    pub account: String,
+pub struct AzblobStorageConfig {
+    /// Account for Azblob
+    #[clap(long = "storage-azblob-account-name", default_value_t)]
+    pub account_name: String,
 
-    /// Master key for Azure storage
-    #[clap(long = "storage-azblob-master-key", default_value_t)]
-    pub master_key: String,
+    /// Master key for Azblob
+    #[clap(long = "storage-azblob-account-key", default_value_t)]
+    pub account_key: String,
 
-    /// Container for Azure storage
+    /// Container for Azblob
     #[clap(long = "storage-azblob-container", default_value_t)]
     pub container: String,
+
+    /// Endpoint URL for Azblob
+    ///
+    /// # TODO(xuanwo)
+    ///
+    /// Clap doesn't allow us to use endpoint_url directly.
+    #[clap(long = "storage-azblob-endpoint-url", default_value_t)]
+    #[serde(rename = "endpoint_url")]
+    pub azblob_endpoint_url: String,
+
+    /// # TODO(xuanwo)
+    ///
+    /// Clap doesn't allow us to use root directly.
+    #[clap(long = "storage-azblob-root", default_value_t)]
+    #[serde(rename = "root")]
+    pub azblob_root: String,
 }
 
-impl fmt::Debug for AzureStorageBlobConfig {
+impl fmt::Debug for AzblobStorageConfig {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         f.debug_struct("AzureStorageBlobConfig")
+            .field("endpoint_url", &self.azblob_endpoint_url)
             .field("container", &self.container)
-            .field("account", &mask_string(&self.account, 3))
-            .field("master_key", &mask_string(&self.master_key, 3))
+            .field("root", &self.azblob_root)
+            .field("account_name", &mask_string(&self.account_name, 3))
+            .field("account_key", &mask_string(&self.account_key, 3))
             .finish()
     }
 }
@@ -166,7 +184,7 @@ pub struct StorageConfig {
 
     // azure storage blob config.
     #[clap(flatten)]
-    pub azure_storage_blob: AzureStorageBlobConfig,
+    pub azblob: AzblobStorageConfig,
 }
 
 impl Default for StorageConfig {
@@ -177,7 +195,7 @@ impl Default for StorageConfig {
 
             fs: FsStorageConfig::default(),
             s3: S3StorageConfig::default(),
-            azure_storage_blob: AzureStorageBlobConfig::default(),
+            azblob: AzblobStorageConfig::default(),
         }
     }
 }
diff --git a/query/src/configs/mod.rs b/query/src/configs/mod.rs
@@ -23,7 +23,7 @@ pub use config::DATABEND_COMMIT_VERSION;
 pub use config_log::LogConfig;
 pub use config_meta::MetaConfig;
 pub use config_query::QueryConfig;
-pub use config_storage::AzureStorageBlobConfig;
+pub use config_storage::AzblobStorageConfig;
 pub use config_storage::FsStorageConfig;
 pub use config_storage::S3StorageConfig;
 pub use config_storage::StorageConfig;
diff --git a/query/src/sessions/session_mgr.rs b/query/src/sessions/session_mgr.rs
@@ -32,6 +32,7 @@ use common_tracing::tracing;
 use common_tracing::tracing_appender::non_blocking::WorkerGuard;
 use futures::future::Either;
 use futures::StreamExt;
+use opendal::services::azblob;
 use opendal::services::fs;
 use opendal::services::memory;
 use opendal::services::s3;
@@ -335,6 +336,41 @@ impl SessionManager {
 
                 builder.finish().await?
             }
+            DalSchema::Azblob => {
+                let azblob_conf = &storage_conf.azblob;
+                let mut builder = azblob::Backend::build();
+
+                // Endpoint
+                {
+                    builder.endpoint(&azblob_conf.azblob_endpoint_url);
+                }
+
+                // Container
+                {
+                    builder.container(&azblob_conf.container);
+                }
+
+                // Root
+                {
+                    builder.root(&azblob_conf.azblob_root);
+                }
+
+                // Credential
+                {
+                    builder.account_name(&azblob_conf.account_name);
+                    builder.account_key(&azblob_conf.account_key);
+                }
+
+                builder.finish().await?
+            }
+            DalSchema::Fs => {
+                let mut path = storage_conf.fs.data_path.clone();
+                if !path.starts_with('/') {
+                    path = env::current_dir().unwrap().join(path).display().to_string();
+                }
+
+                fs::Backend::build().root(&path).finish().await?
+            }
             DalSchema::S3 => {
                 let s3_conf = &storage_conf.s3;
                 let mut builder = s3::Backend::build();
@@ -362,22 +398,11 @@ impl SessionManager {
 
                 // Root.
                 {
-                    if !s3_conf.root.is_empty() {
-                        builder.root(&s3_conf.root);
-                    }
+                    builder.root(&s3_conf.root);
                 }
 
                 builder.finish().await?
             }
-            DalSchema::Fs => {
-                let mut path = storage_conf.fs.data_path.clone();
-                if !path.starts_with('/') {
-                    path = env::current_dir().unwrap().join(path).display().to_string();
-                }
-
-                fs::Backend::build().root(&path).finish().await?
-            }
-            _ => return Err(ErrorCode::StorageOther("not supported storage backend")),
         };
 
         Ok(Operator::new(accessor))

diff --git a/query/src/storages/system/configs_table.rs b/query/src/storages/system/configs_table.rs
@@ -87,6 +87,8 @@ impl SyncSystemTable for ConfigsTable {
         let mut storage_config = config.storage;
         storage_config.s3.access_key_id = mask_string(&storage_config.s3.access_key_id, 3);
         storage_config.s3.secret_access_key = mask_string(&storage_config.s3.secret_access_key, 3);
+        storage_config.azblob.account_name = mask_string(&storage_config.azblob.account_name, 3);
+        storage_config.azblob.account_key = mask_string(&storage_config.azblob.account_key, 3);
         let storage_config_value = serde_json::to_value(storage_config)?;
         ConfigsTable::extract_config(
             &mut names,

diff --git a/query/tests/it/configs.rs b/query/tests/it/configs.rs
@@ -97,10 +97,12 @@ secret_access_key = ""
 bucket = ""
 root = ""
 
-[storage.azure_storage_blob]
-account = ""
-master_key = ""
+[storage.azblob]
+account_name = ""
+account_key = ""
 container = ""
+endpoint_url = ""
+root = ""
 "#;
 
     let tom_actual = toml::to_string(&actual).unwrap();
@@ -267,10 +269,12 @@ secret_access_key = ""
 bucket = ""
 root = ""
 
-[storage.azure_storage_blob]
-account = ""
-master_key = ""
+[storage.azblob]
+account_name = ""
+account_key = ""
 container = ""
+endpoint_url = ""
+root = ""
     "#
         .as_bytes(),
     )?;

diff --git a/query/tests/it/storages/system/configs_table.rs b/query/tests/it/storages/system/configs_table.rs
@@ -86,9 +86,11 @@ async fn test_configs_table() -> Result<()> {
         "| query   | table_memory_cache_mb_size           | 256                      |             |",
         "| query   | tenant_id                            | test                     |             |",
         "| query   | wait_timeout_mills                   | 5000                     |             |",
-        "| storage | azure_storage_blob.account           |                          |             |",
-        "| storage | azure_storage_blob.container         |                          |             |",
-        "| storage | azure_storage_blob.master_key        |                          |             |",
+        "| storage | azblob.account_key                   |                          |             |",
+        "| storage | azblob.account_name                  |                          |             |",
+        "| storage | azblob.container                     |                          |             |",
+        "| storage | azblob.endpoint_url                  |                          |             |",
+        "| storage | azblob.root                          |                          |             |",
         "| storage | fs.data_path                         | _data                    |             |",
         "| storage | num_cpus                             | 0                        |             |",
         "| storage | s3.access_key_id                     |                          |             |",
@@ -173,9 +175,11 @@ async fn test_configs_table_redact() -> Result<()> {
         "| query   | table_memory_cache_mb_size           | 256                      |             |",
         "| query   | tenant_id                            | test                     |             |",
         "| query   | wait_timeout_mills                   | 5000                     |             |",
-        "| storage | azure_storage_blob.account           |                          |             |",
-        "| storage | azure_storage_blob.container         |                          |             |",
-        "| storage | azure_storage_blob.master_key        |                          |             |",
+        "| storage | azblob.account_key                   |                          |             |",
+        "| storage | azblob.account_name                  |                          |             |",
+        "| storage | azblob.container                     |                          |             |",
+        "| storage | azblob.endpoint_url                  |                          |             |",
+        "| storage | azblob.root                          |                          |             |",
         "| storage | fs.data_path                         | _data                    |             |",
         "| storage | num_cpus                             | 0                        |             |",
         "| storage | s3.access_key_id                     | ******_id                |             |",

diff --git a/scripts/ci/deploy/config/databend-query-embedded-meta.toml b/scripts/ci/deploy/config/databend-query-embedded-meta.toml
@@ -60,4 +60,4 @@ data_path = "stateless_test_data"
 [storage.s3]
 
 # Azure storage
-[storage.azure_storage_blob]
+[storage.azblob]
diff --git a/scripts/ci/deploy/config/databend-query-management-mode.toml b/scripts/ci/deploy/config/databend-query-management-mode.toml
@@ -55,4 +55,4 @@ data_path = "management_data"
 [storage.s3]
 
 # Azure storage
-[storage.azure_storage_blob]
+[storage.azblob]
diff --git a/scripts/ci/deploy/config/databend-query-node-1.toml b/scripts/ci/deploy/config/databend-query-node-1.toml
@@ -66,4 +66,4 @@ data_path = "stateless_test_data"
 [storage.s3]
 
 # Azure storage
-[storage.azure_storage_blob]
+[storage.azblob]
diff --git a/scripts/ci/deploy/config/databend-query-node-2.toml b/scripts/ci/deploy/config/databend-query-node-2.toml
@@ -65,4 +65,4 @@ data_path = "stateless_test_data"
 [storage.s3]
 
 # Azure storage
-[storage.azure_storage_blob]
+[storage.azblob]
diff --git a/scripts/ci/deploy/config/databend-query-node-3.toml b/scripts/ci/deploy/config/databend-query-node-3.toml
@@ -65,4 +65,4 @@ data_path = "stateless_test_data"
 [storage.s3]
 
 # Azure storage
-[storage.azure_storage_blob]
+[storage.azblob]