move Clickhouse HTTP handler to its own port.

docker/databend-query-docker.toml

@@ -23,6 +23,10 @@ mysql_handler_port = 3307
 clickhouse_handler_host = "0.0.0.0"
 clickhouse_handler_port = 9001
 
+# Databend Query ClickHouse HTTP Handler.
+clickhouse_http_handler_host = "0.0.0.0"
+clickhouse_http_handler_port = 8125
+
 # Databend Query HTTP Handler.
 http_handler_host = "0.0.0.0"
 http_handler_port = 8001

docs/doc/10-deploy/00-local.md

@@ -140,6 +140,10 @@ mysql_handler_port = 3307
 clickhouse_handler_host = "127.0.0.1"
 clickhouse_handler_port = 9001
 
+# Query ClickHouse HTTP Handler.
+clickhouse_http_handler_host = "127.0.0.1"
+clickhouse_http_handler_port = 8125
+
 # Query HTTP Handler.
 http_handler_host = "127.0.0.1"
 http_handler_port = 8081

docs/doc/10-deploy/01-s3.md

@@ -106,6 +106,10 @@ mysql_handler_port = 3307
 clickhouse_handler_host = "127.0.0.1"
 clickhouse_handler_port = 9001
 
+# Query ClickHouse HTTP Handler.
+clickhouse_http_handler_host = "127.0.0.1"
+clickhouse_http_handler_port = 8125
+
 # Query HTTP Handler.
 http_handler_host = "127.0.0.1"
 http_handler_port = 8081

docs/doc/10-deploy/02-cos.md

@@ -113,6 +113,10 @@ mysql_handler_port = 3307
 clickhouse_handler_host = "127.0.0.1"
 clickhouse_handler_port = 9001
 
+# Query ClickHouse HTTP Handler.
+clickhouse_http_handler_host = "127.0.0.1"
+clickhouse_http_handler_port = 8125
+
 # Query HTTP Handler.
 http_handler_host = "127.0.0.1"
 http_handler_port = 8081

docs/doc/10-deploy/03-oss.md

@@ -114,6 +114,10 @@ mysql_handler_port = 3307
 clickhouse_handler_host = "127.0.0.1"
 clickhouse_handler_port = 9001
 
+# Query ClickHouse HTTP Handler.
+clickhouse_http_handler_host = "127.0.0.1"
+clickhouse_http_handler_port = 8125
+
 # Query HTTP Handler.
 http_handler_host = "127.0.0.1"
 http_handler_port = 8081

docs/doc/10-deploy/05-wasabi.md

@@ -117,6 +117,10 @@ mysql_handler_port = 3307
 clickhouse_handler_host = "127.0.0.1"
 clickhouse_handler_port = 9001
 
+# Query ClickHouse HTTP Handler.
+clickhouse_http_handler_host = "127.0.0.1"
+clickhouse_http_handler_port = 8125
+
 # Query HTTP Handler.
 http_handler_host = "127.0.0.1"
 http_handler_port = 8081

docs/doc/10-deploy/06-scw.md

@@ -113,6 +113,10 @@ mysql_handler_port = 3307
 clickhouse_handler_host = "127.0.0.1"
 clickhouse_handler_port = 9001
 
+# Query ClickHouse HTTP Handler.
+clickhouse_http_handler_host = "127.0.0.1"
+clickhouse_http_handler_port = 8125
+
 # Query HTTP Handler.
 http_handler_host = "127.0.0.1"
 http_handler_port = 8081

docs/doc/10-deploy/07-qingstore.md

@@ -113,6 +113,10 @@ mysql_handler_port = 3307
 clickhouse_handler_host = "127.0.0.1"
 clickhouse_handler_port = 9001
 
+# Query ClickHouse HTTP Handler.
+clickhouse_http_handler_host = "127.0.0.1"
+clickhouse_http_handler_port = 8125
+
 # Query HTTP Handler.
 http_handler_host = "127.0.0.1"
 http_handler_port = 8081

docs/doc/10-deploy/08-azure.md

@@ -113,6 +113,10 @@ mysql_handler_port = 3307
 clickhouse_handler_host = "127.0.0.1"
 clickhouse_handler_port = 9001
 
+# Query ClickHouse HTTP Handler.
+clickhouse_http_handler_host = "127.0.0.1"
+clickhouse_http_handler_port = 8125
+
 # Query HTTP Handler.
 http_handler_host = "127.0.0.1"
 http_handler_port = 8081

docs/doc/10-deploy/10-cluster-minio.md

@@ -57,6 +57,10 @@ mysql_handler_port = 3308
 clickhouse_handler_host = "127.0.0.1"
 clickhouse_handler_port = 9002
 
+# Query ClickHouse HTTP Handler.
+clickhouse_http_handler_host = "127.0.0.1"
+clickhouse_http_handler_port = 8125
+
 # Query HTTP Handler.
 http_handler_host = "127.0.0.1"
 http_handler_port = 8082

docs/doc/30-reference/30-sql/70-system-tables/system-tracing.md

@@ -9,7 +9,7 @@ SELECT * FROM system.tracing LIMIT 1\G
 *************************** 1. row ***************************
        v: 0
     name: databend-query-test_cluster@0.0.0.0:3307
-     msg: Config { config_file: "scripts/ci/deploy/config/databend-query-node-1.toml", query: QueryConfig { tenant_id: "test_tenant", cluster_id: "test_cluster", num_cpus: 10, mysql_handler_host: "0.0.0.0", mysql_handler_port: 3307, max_active_sessions: 256, clickhouse_handler_host: "0.0.0.0", clickhouse_handler_port: 9001, http_handler_host: "0.0.0.0", http_handler_port: 8001, http_handler_result_timeout_millis: 10000, flight_api_address: "0.0.0.0:9091", admin_api_address: "0.0.0.0:8081", metric_api_address: "0.0.0.0:7071", http_handler_tls_server_cert: "", http_handler_tls_server_key: "", http_handler_tls_server_root_ca_cert: "", api_tls_server_cert: "", api_tls_server_key: "", api_tls_server_root_ca_cert: "", rpc_tls_server_cert: "", rpc_tls_server_key: "", rpc_tls_query_server_root_ca_cert: "", rpc_tls_query_service_domain_name: "localhost", table_engine_memory_enabled: true, database_engine_github_enabled: true, wait_timeout_mills: 5000, max_query_log_size: 10000, table_cache_enabled: true, table_cache_snapshot_count: 256, table_cache_segment_count: 10240, table_cache_block_meta_count: 102400, table_memory_cache_mb_size: 1024, table_disk_cache_root: "_cache", table_disk_cache_mb_size: 10240, management_mode: false, jwt_key_file: "" }, log: LogConfig { log_level: "INFO", log_dir: "./_logs", log_query_enabled: false }, meta: {meta_address: "0.0.0.0:9191", meta_user: "root", meta_password: "******"}, storage: StorageConfig { storage_type: "disk", storage_num_cpus: 0, disk: FsStorageConfig { data_path: "stateless_test_data", temp_data_path: "" }, s3: {s3.storage.region: "", s3.storage.endpoint_url: "https://s3.amazonaws.com", s3.storage.bucket: "", s3.storage.access_key_id: "", s3.storage.secret_access_key: "", }, azure_storage_blob: {Azure.storage.container: "", } } }
+     msg: Config { config_file: "scripts/ci/deploy/config/databend-query-node-1.toml", query: QueryConfig { tenant_id: "test_tenant", cluster_id: "test_cluster", num_cpus: 10, mysql_handler_host: "0.0.0.0", mysql_handler_port: 3307, max_active_sessions: 256, clickhouse_handler_host: "0.0.0.0", clickhouse_handler_port: 9001, clickhouse_http_handler_host: "0.0.0.0", clickhouse_http_handler_port: 8125, http_handler_host: "0.0.0.0", http_handler_port: 8001, http_handler_result_timeout_millis: 10000, flight_api_address: "0.0.0.0:9091", admin_api_address: "0.0.0.0:8081", metric_api_address: "0.0.0.0:7071", http_handler_tls_server_cert: "", http_handler_tls_server_key: "", http_handler_tls_server_root_ca_cert: "", api_tls_server_cert: "", api_tls_server_key: "", api_tls_server_root_ca_cert: "", rpc_tls_server_cert: "", rpc_tls_server_key: "", rpc_tls_query_server_root_ca_cert: "", rpc_tls_query_service_domain_name: "localhost", table_engine_memory_enabled: true, database_engine_github_enabled: true, wait_timeout_mills: 5000, max_query_log_size: 10000, table_cache_enabled: true, table_cache_snapshot_count: 256, table_cache_segment_count: 10240, table_cache_block_meta_count: 102400, table_memory_cache_mb_size: 1024, table_disk_cache_root: "_cache", table_disk_cache_mb_size: 10240, management_mode: false, jwt_key_file: "" }, log: LogConfig { log_level: "INFO", log_dir: "./_logs", log_query_enabled: false }, meta: {meta_address: "0.0.0.0:9191", meta_user: "root", meta_password: "******"}, storage: StorageConfig { storage_type: "disk", storage_num_cpus: 0, disk: FsStorageConfig { data_path: "stateless_test_data", temp_data_path: "" }, s3: {s3.storage.region: "", s3.storage.endpoint_url: "https://s3.amazonaws.com", s3.storage.bucket: "", s3.storage.access_key_id: "", s3.storage.secret_access_key: "", }, azure_storage_blob: {Azure.storage.container: "", } } }
    level: 30
 hostname: localhost
      pid: 24640

docs/doc/50-manage/01-query/10-query-config.md

@@ -94,10 +94,22 @@ If `endpoints` is configured, the `address` configuration will no longer take ef
 
 ### clickhouse_handler_port
 
-* The port to listen on for ClickHouse handler, e.g., `3307`.
+* The port to listen on for ClickHouse handler, e.g., `9000`.
 * Default: `9000`
 * Env variable: `QUERY_CLICKHOUSE_HANDLER_PORT`
 
+### clickhouse_http_handler_host
+
+* The IP address to listen on for ClickHouse HTTP handler, e.g., `0.0.0.0`.
+* Default: `"127.0.0.1"`
+* Env variable: `QUERY_CLICKHOUSE_HTTP_HANDLER_HOST`
+
+### clickhouse_http_handler_port
+
+* The port to listen on for ClickHouse HTTP handler, e.g., `8124`.
+* Default: `8124`
+* Env variable: `QUERY_CLICKHOUSE_HTTP_HANDLER_PORT`
+
 ### tenant_id
 
 * The ID for the databend-query server to store metadata to the Meta Service.

query/bin/databend-query.rs

@@ -28,6 +28,7 @@ use databend_query::api::RpcService;
 use databend_query::metrics::MetricService;
 use databend_query::servers::ClickHouseHandler;
 use databend_query::servers::HttpHandler;
+use databend_query::servers::HttpHandlerKind;
 use databend_query::servers::MySQLHandler;
 use databend_query::servers::Server;
 use databend_query::servers::ShutdownHandle;
@@ -105,18 +106,35 @@ async fn main(_global_tracker: Arc<RuntimeTracker>) -> common_exception::Result<
         let hostname = conf.query.http_handler_host.clone();
         let listening = format!("{}:{}", hostname, conf.query.http_handler_port);
 
-        let mut srv = HttpHandler::create(session_manager.clone());
+        let mut srv = HttpHandler::create(session_manager.clone(), HttpHandlerKind::Query);
         let listening = srv.start(listening.parse()?).await?;
         shutdown_handle.add_service(srv);
 
-        let http_handler_usage = HttpHandler::usage(listening);
+        let http_handler_usage = HttpHandlerKind::Query.usage(listening);
         tracing::info!(
             "Http handler listening on {} {}",
             listening,
             http_handler_usage
         );
     }
 
+    // clickhouse HTTP handler.
+    {
+        let hostname = conf.query.clickhouse_http_handler_host.clone();
+        let listening = format!("{}:{}", hostname, conf.query.clickhouse_http_handler_port);
+
+        let mut srv = HttpHandler::create(session_manager.clone(), HttpHandlerKind::Clickhouse);
+        let listening = srv.start(listening.parse()?).await?;
+        shutdown_handle.add_service(srv);
+
+        let http_handler_usage = HttpHandlerKind::Clickhouse.usage(listening);
+        tracing::info!(
+            "clickhouse Http handler listening on {} {}",
+            listening,
+            http_handler_usage
+        );
+    }
+
     // Metric API service.
     {
         let address = conf.query.metric_api_address.clone();

query/src/config/inner.rs

@@ -104,6 +104,8 @@ pub struct QueryConfig {
     pub max_active_sessions: u64,
     pub clickhouse_handler_host: String,
     pub clickhouse_handler_port: u16,
+    pub clickhouse_http_handler_host: String,
+    pub clickhouse_http_handler_port: u16,
     pub http_handler_host: String,
     pub http_handler_port: u16,
     pub http_handler_result_timeout_millis: u64,
@@ -165,6 +167,8 @@ impl Default for QueryConfig {
             max_active_sessions: 256,
             clickhouse_handler_host: "127.0.0.1".to_string(),
             clickhouse_handler_port: 9000,
+            clickhouse_http_handler_host: "127.0.0.1".to_string(),
+            clickhouse_http_handler_port: 8124,
             http_handler_host: "127.0.0.1".to_string(),
             http_handler_port: 8000,
             http_handler_result_timeout_millis: 10000,

query/src/config/outer_v0.rs

@@ -516,6 +516,12 @@ pub struct QueryConfig {
     #[clap(long, default_value = "9000")]
     pub clickhouse_handler_port: u16,
 
+    #[clap(long, default_value = "127.0.0.1")]
+    pub clickhouse_http_handler_host: String,
+
+    #[clap(long, default_value = "8124")]
+    pub clickhouse_http_handler_port: u16,
+
     #[clap(long, default_value = "127.0.0.1")]
     pub http_handler_host: String,
 
@@ -660,6 +666,8 @@ impl TryInto<InnerQueryConfig> for QueryConfig {
             max_active_sessions: self.max_active_sessions,
             clickhouse_handler_host: self.clickhouse_handler_host,
             clickhouse_handler_port: self.clickhouse_handler_port,
+            clickhouse_http_handler_host: self.clickhouse_http_handler_host,
+            clickhouse_http_handler_port: self.clickhouse_http_handler_port,
             http_handler_host: self.http_handler_host,
             http_handler_port: self.http_handler_port,
             http_handler_result_timeout_millis: self.http_handler_result_timeout_millis,
@@ -710,6 +718,8 @@ impl From<InnerQueryConfig> for QueryConfig {
             max_active_sessions: inner.max_active_sessions,
             clickhouse_handler_host: inner.clickhouse_handler_host,
             clickhouse_handler_port: inner.clickhouse_handler_port,
+            clickhouse_http_handler_host: inner.clickhouse_http_handler_host,
+            clickhouse_http_handler_port: inner.clickhouse_http_handler_port,
             http_handler_host: inner.http_handler_host,
             http_handler_port: inner.http_handler_port,
             http_handler_result_timeout_millis: inner.http_handler_result_timeout_millis,

query/src/servers/http/http_services.rs

@@ -35,43 +35,68 @@ use crate::servers::Server;
 use crate::sessions::SessionManager;
 use crate::Config;
 
+pub enum HttpHandlerKind {
+    Query,
+    Clickhouse,
+}
+
+impl HttpHandlerKind {
+    pub fn usage(&self, sock: SocketAddr) -> String {
+        match self {
+            HttpHandlerKind::Query => {
+                let json = r#"{"foo": "bar"}"#;
+                format!(
+                    r#" examples:
+curl -u root: --request POST '{:?}/v1/query/' --header 'Content-Type: application/json' --data-raw '{{"sql": "SELECT avg(number) FROM numbers(100000000)"}}'
+echo '{}' "#,
+                    sock, json,
+                )
+            }
+            HttpHandlerKind::Clickhouse => {
+                format!(
+                    r#" examples:
+curl '{:?}/clickhouse/?query=INSERT%20INTO%20test%20FORMAT%20JSONEachRow' --data-binary @-"#,
+                    sock,
+                )
+            }
+        }
+    }
+}
+
 pub struct HttpHandler {
     session_manager: Arc<SessionManager>,
     shutdown_handler: HttpShutdownHandler,
+    kind: HttpHandlerKind,
 }
 
 impl HttpHandler {
-    pub fn create(session_manager: Arc<SessionManager>) -> Box<dyn Server> {
+    pub fn create(session_manager: Arc<SessionManager>, kind: HttpHandlerKind) -> Box<dyn Server> {
         Box::new(HttpHandler {
             session_manager,
             shutdown_handler: HttpShutdownHandler::create("http handler".to_string()),
+            kind,
         })
     }
 
-    pub fn usage(sock: SocketAddr) -> String {
-        let json = r#"{"foo": "bar"}"#;
-        format!(
-            r#" examples:
-curl -u root: --request POST '{:?}/v1/query/' --header 'Content-Type: application/json' --data-raw '{{"sql": "SELECT avg(number) FROM numbers(100000000)"}}'
-echo '{}' | curl '{:?}/clickhouse/?query=INSERT%20INTO%20test%20FORMAT%20JSONEachRow' --data-binary @-"#,
-            sock, json, sock
-        )
-    }
-
     fn build_router(&self, sock: SocketAddr) -> impl Endpoint {
-        Route::new()
-            .at(
-                "/",
-                get(poem::endpoint::make_sync(move |_| Self::usage(sock))),
-            )
-            .nest("/clickhouse", clickhouse_router())
-            .nest("/v1/query", query_route())
-            .at("/v1/streaming_load", put(streaming_load))
-            .at("/v1/upload_to_stage", put(upload_to_stage))
-            .with(HTTPSessionMiddleware {
-                session_manager: self.session_manager.clone(),
-            })
-            .boxed()
+        let ep = match self.kind {
+            HttpHandlerKind::Query => Route::new()
+                .at(
+                    "/",
+                    get(poem::endpoint::make_sync(move |_| {
+                        HttpHandlerKind::Query.usage(sock)
+                    })),
+                )
+                .nest("/clickhouse", clickhouse_router())
+                .nest("/v1/query", query_route())
+                .at("/v1/streaming_load", put(streaming_load))
+                .at("/v1/upload_to_stage", put(upload_to_stage)),
+            HttpHandlerKind::Clickhouse => Route::new().nest("/", clickhouse_router()),
+        };
+        ep.with(HTTPSessionMiddleware {
+            session_manager: self.session_manager.clone(),
+        })
+        .boxed()
     }
 
     fn build_tls(config: &Config) -> Result<RustlsConfig> {

query/src/servers/http/mod.rs

@@ -19,3 +19,4 @@ pub mod middleware;
 pub mod v1;
 
 pub use http_services::HttpHandler;
+pub use http_services::HttpHandlerKind;

query/src/servers/mod.rs

@@ -19,6 +19,7 @@ pub use server::Server;
 pub use server::ShutdownHandle;
 
 pub use self::http::HttpHandler;
+pub use self::http::HttpHandlerKind;
 pub use self::mysql::MySQLConnection;
 pub use self::mysql::MySQLFederated;
 pub use self::mysql::MySQLHandler;

query/tests/it/configs.rs

@@ -37,6 +37,8 @@ mysql_handler_port = 3307
 max_active_sessions = 256
 clickhouse_handler_host = "127.0.0.1"
 clickhouse_handler_port = 9000
+clickhouse_http_handler_host = "127.0.0.1"
+clickhouse_http_handler_port = 8124
 http_handler_host = "127.0.0.1"
 http_handler_port = 8000
 http_handler_result_timeout_millis = 10000
@@ -138,6 +140,8 @@ fn test_env_config_s3() -> Result<()> {
             ("QUERY_MAX_ACTIVE_SESSIONS", Some("255")),
             ("QUERY_CLICKHOUSE_HANDLER_HOST", Some("1.2.3.4")),
             ("QUERY_CLICKHOUSE_HANDLER_PORT", Some("9000")),
+            ("QUERY_CLICKHOUSE_HTTP_HANDLER_HOST", Some("1.2.3.4")),
+            ("QUERY_CLICKHOUSE_HTTP_HANDLER_PORT", Some("8124")),
             ("QUERY_HTTP_HANDLER_HOST", Some("1.2.3.4")),
             ("QUERY_HTTP_HANDLER_PORT", Some("8001")),
             ("QUERY_FLIGHT_API_ADDRESS", Some("1.2.3.4:9091")),
@@ -171,6 +175,8 @@ fn test_env_config_s3() -> Result<()> {
             assert_eq!(255, configured.query.max_active_sessions);
             assert_eq!("1.2.3.4", configured.query.clickhouse_handler_host);
             assert_eq!(9000, configured.query.clickhouse_handler_port);
+            assert_eq!("1.2.3.4", configured.query.clickhouse_http_handler_host);
+            assert_eq!(8124, configured.query.clickhouse_http_handler_port);
             assert_eq!("1.2.3.4", configured.query.http_handler_host);
             assert_eq!(8001, configured.query.http_handler_port);
 
@@ -216,6 +222,8 @@ fn test_env_config_fs() -> Result<()> {
             ("QUERY_MAX_ACTIVE_SESSIONS", Some("255")),
             ("QUERY_CLICKHOUSE_HANDLER_HOST", Some("1.2.3.4")),
             ("QUERY_CLICKHOUSE_HANDLER_PORT", Some("9000")),
+            ("QUERY_CLICKHOUSE_HTTP_HANDLER_HOST", Some("1.2.3.4")),
+            ("QUERY_CLICKHOUSE_HTTP_HANDLER_PORT", Some("8124")),
             ("QUERY_HTTP_HANDLER_HOST", Some("1.2.3.4")),
             ("QUERY_HTTP_HANDLER_PORT", Some("8001")),
             ("QUERY_FLIGHT_API_ADDRESS", Some("1.2.3.4:9091")),
@@ -249,6 +257,8 @@ fn test_env_config_fs() -> Result<()> {
             assert_eq!(255, configured.query.max_active_sessions);
             assert_eq!("1.2.3.4", configured.query.clickhouse_handler_host);
             assert_eq!(9000, configured.query.clickhouse_handler_port);
+            assert_eq!("1.2.3.4", configured.query.clickhouse_http_handler_host);
+            assert_eq!(8124, configured.query.clickhouse_http_handler_port);
             assert_eq!("1.2.3.4", configured.query.http_handler_host);
             assert_eq!(8001, configured.query.http_handler_port);
 
@@ -299,6 +309,8 @@ mysql_handler_port = 3307
 max_active_sessions = 256
 clickhouse_handler_host = "127.0.0.1"
 clickhouse_handler_port = 9000
+clickhouse_http_handler_host = "127.0.0.1"
+clickhouse_http_handler_port = 8124
 http_handler_host = "127.0.0.1"
 http_handler_port = 8000
 http_handler_result_timeout_millis = 10000