From 5137d7f77dbd7c5281a924988292739e334aaa8b Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Tue, 16 Jul 2024 15:54:53 +0300 Subject: [PATCH] Merge #5932 #5984 #6257 #6298 #6471 #6484 #6703 (#6721) Co-authored-by: Timur Sufiyanov --- .../external_sources/external_data_source.cpp | 8 ++ .../libs/actors/clusters_from_connections.cpp | 24 ++++++ ydb/core/fq/libs/actors/database_resolver.cpp | 56 ++++++++++++- .../libs/actors/ut/database_resolver_ut.cpp | 81 ++++++++++++++++++- ydb/core/fq/libs/common/util.cpp | 22 +++++ ydb/core/fq/libs/compute/common/config.h | 2 + .../actors/query_utils.cpp | 37 ++++++++- .../fq/libs/control_plane_proxy/utils/utils.h | 6 ++ .../request_validators.cpp | 40 ++++++++- .../ydb_control_plane_storage_queries.cpp | 4 + .../mdb_endpoint_generator.cpp | 12 ++- .../kqp/provider/yql_kikimr_gateway_ut.cpp | 2 + .../ut_external_data_source.cpp | 4 + ydb/public/api/protos/draft/fq.proto | 21 +++++ 14 files changed, 306 insertions(+), 13 deletions(-) diff --git a/ydb/core/external_sources/external_data_source.cpp b/ydb/core/external_sources/external_data_source.cpp index e11c7cce2446..d44c8ca6dbb1 100644 --- a/ydb/core/external_sources/external_data_source.cpp +++ b/ydb/core/external_sources/external_data_source.cpp @@ -36,6 +36,10 @@ struct TExternalDataSource : public IExternalSource { ythrow TExternalSourceException() << "Only external table supports parameters"; } + bool IsRDBMSDataSource(const TProtoStringType& sourceType) const { + return IsIn({"Greenplum", "PostgreSQL", "MySQL", "MsSQLServer", "Clickhouse"}, sourceType); + } + virtual void ValidateExternalDataSource(const TString& externalDataSourceDescription) const override { NKikimrSchemeOp::TExternalDataSourceDescription proto; if (!proto.ParseFromString(externalDataSourceDescription)) { @@ -49,6 +53,10 @@ struct TExternalDataSource : public IExternalSource { ythrow TExternalSourceException() << "Unsupported property: " << key; } + if (IsRDBMSDataSource(proto.GetSourceType()) && !proto.GetProperties().GetProperties().contains("database_name")){ + ythrow TExternalSourceException() << proto.GetSourceType() << " source must provide database_name"; + } + ValidateHostname(HostnamePatterns, proto.GetLocation()); } diff --git a/ydb/core/fq/libs/actors/clusters_from_connections.cpp b/ydb/core/fq/libs/actors/clusters_from_connections.cpp index 066668c574a9..7b2d8187fd1e 100644 --- a/ydb/core/fq/libs/actors/clusters_from_connections.cpp +++ b/ydb/core/fq/libs/actors/clusters_from_connections.cpp @@ -271,6 +271,30 @@ void AddClustersFromConnections( clusters.emplace(connectionName, GenericProviderName); break; } + case FederatedQuery::ConnectionSetting::kGreenplumCluster: { + FillGenericClusterConfig( + common, + *gatewaysConfig.MutableGeneric()->AddClusterMapping(), + conn.content().setting().greenplum_cluster(), + connectionName, + NYql::NConnector::NApi::EDataSourceKind::GREENPLUM, + authToken, + accountIdSignatures); + clusters.emplace(connectionName, GenericProviderName); + break; + } + case FederatedQuery::ConnectionSetting::kMysqlCluster: { + FillGenericClusterConfig( + common, + *gatewaysConfig.MutableGeneric()->AddClusterMapping(), + conn.content().setting().mysql_cluster(), + connectionName, + NYql::NConnector::NApi::EDataSourceKind::MYSQL, + authToken, + accountIdSignatures); + clusters.emplace(connectionName, GenericProviderName); + break; + } // Do not replace with default. Adding a new connection should cause a compilation error case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET: diff --git a/ydb/core/fq/libs/actors/database_resolver.cpp b/ydb/core/fq/libs/actors/database_resolver.cpp index d098a1ed76a4..74ba4e3ddf89 100644 --- a/ydb/core/fq/libs/actors/database_resolver.cpp +++ b/ydb/core/fq/libs/actors/database_resolver.cpp @@ -319,7 +319,7 @@ class TDatabaseResolver: public TActor // There are two kinds of managed YDBs: serverless and dedicated. // While working with dedicated databases, we have to use underlay network. // That's why we add `u-` prefix to database fqdn. - if (databaseInfo.GetMap().contains("dedicatedDatabase")) { + if (databaseInfo.GetMap().contains("storageConfig")) { endpoint = "u-" + endpoint; host = "u-" + host; } @@ -335,7 +335,7 @@ class TDatabaseResolver: public TActor { auto ret = ydbParser(databaseInfo, mdbEndpointGenerator, useTls, protocol); // TODO: Take explicit field from MVP - bool isDedicatedDb = databaseInfo.GetMap().contains("dedicatedDatabase"); + bool isDedicatedDb = databaseInfo.GetMap().contains("storageConfig"); if (!isDedicatedDb && ret.Endpoint.StartsWith("ydb.")) { // Replace "ydb." -> "yds." ret.Endpoint[2] = 's'; @@ -457,6 +457,56 @@ class TDatabaseResolver: public TActor endpoint = mdbEndpointGenerator->ToEndpoint(params); + return TDatabaseDescription{"", endpoint.first, endpoint.second, "", useTls}; + }; + Parsers[NYql::EDatabaseType::MySQL] = []( + NJson::TJsonValue& databaseInfo, + const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, + bool useTls, + NConnector::NApi::EProtocol protocol + ) { + NYql::IMdbEndpointGenerator::TEndpoint endpoint; + TVector aliveHosts; + + const auto& hostsArray = databaseInfo.GetMap().at("hosts").GetArraySafe(); + + for (const auto& host : hostsArray) { + const auto& hostMap = host.GetMap(); + + if (!hostMap.contains("services")) { + // indicates that cluster is down + continue; + } + + const auto& servicesArray = hostMap.at("services").GetArraySafe(); + + // check if all services of a particular host are alive + const bool alive = std::all_of( + servicesArray.begin(), + servicesArray.end(), + [](const auto& service) { + return service["health"].GetString() == "ALIVE"; + } + ); + + if (alive) { + aliveHosts.push_back(host["name"].GetString()); + } + } + + if (aliveHosts.empty()) { + ythrow TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "No ALIVE MySQL hosts found"; + } + + NYql::IMdbEndpointGenerator::TParams params = { + .DatabaseType = NYql::EDatabaseType::MySQL, + .MdbHost = aliveHosts[std::rand() % static_cast(aliveHosts.size())], + .UseTls = useTls, + .Protocol = protocol, + }; + + endpoint = mdbEndpointGenerator->ToEndpoint(params); + return TDatabaseDescription{"", endpoint.first, endpoint.second, "", useTls}; }; } @@ -538,7 +588,7 @@ class TDatabaseResolver: public TActor url = TUrlBuilder(ev->Get()->YdbMvpEndpoint + "/database") .AddUrlParam("databaseId", databaseId) .Build(); - } else if (IsIn({NYql::EDatabaseType::ClickHouse, NYql::EDatabaseType::PostgreSQL}, databaseType)) { + } else if (IsIn({NYql::EDatabaseType::ClickHouse, NYql::EDatabaseType::PostgreSQL, NYql::EDatabaseType::MySQL}, databaseType)) { YQL_ENSURE(ev->Get()->MdbGateway, "empty MDB Gateway"); url = TUrlBuilder( ev->Get()->MdbGateway + "/managed-" + NYql::DatabaseTypeLowercase(databaseType) + "/v1/clusters/") diff --git a/ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp b/ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp index 2a8c0daa98b1..f4e26ce40a7b 100644 --- a/ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp +++ b/ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp @@ -242,7 +242,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { R"( { "endpoint":"grpcs://lb.etnbrtlini51k7cinbdr.ydb.mdb.yandexcloud.net:2135/?database=/ru-central1/b1gtl2kg13him37quoo6/etn021us5r9rhld1vgbh", - "dedicatedDatabase":{"resuorcePresetId": "medium"} + "storageConfig":{"storageSizeLimit":107374182400} })", NYql::TDatabaseResolverResponse::TDatabaseDescription{ TString{"u-lb.etnbrtlini51k7cinbdr.ydb.mdb.yandexcloud.net:2135"}, @@ -285,7 +285,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { R"( { "endpoint":"grpcs://lb.etn021us5r9rhld1vgbh.ydb.mdb.yandexcloud.net:2135/?database=/ru-central1/b1g7jdjqd07qg43c4fmp/etn021us5r9rhld1vgbh", - "dedicatedDatabase":{"resourcePresetId": "medium"} + "storageConfig":{"storageSizeLimit":107374182400} })", NYql::TDatabaseResolverResponse::TDatabaseDescription{ TString{"u-lb.etn021us5r9rhld1vgbh.ydb.mdb.yandexcloud.net:2135"}, @@ -473,6 +473,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { issues ); } + Y_UNIT_TEST(Greenplum_MasterNode) { Test( NYql::EDatabaseType::Greenplum, @@ -504,7 +505,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { TString(""), true}, {}); - } + } Y_UNIT_TEST(Greenplum_PermissionDenied) { NYql::TIssues issues{ @@ -535,7 +536,79 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { )", NYql::TDatabaseResolverResponse::TDatabaseDescription{}, issues); - } + } + + Y_UNIT_TEST(MySQL) { + Test( + NYql::EDatabaseType::MySQL, + NYql::NConnector::NApi::EProtocol::NATIVE, + "https://mdb.api.cloud.yandex.net:443/managed-mysql/v1/clusters/etn021us5r9rhld1vgbh/hosts", + "200", + R"({ + "hosts": [ + { + "services": [ + { + "type": "POOLER", + "health": "ALIVE" + }, + { + "type": "MYSQL", + "health": "ALIVE" + } + ], + "name": "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", + "clusterId": "c9qb2bjghs8onbncpamk", + "zoneId": "ru-central1-b", + "role": "MASTER", + "health": "ALIVE" + } + ] + })", + NYql::TDatabaseResolverResponse::TDatabaseDescription{ + TString{""}, + TString{"rc1b-eyt6dtobu96rwydq.db.yandex.net"}, + 3306, + TString(""), + true + }, + {}); + } + + Y_UNIT_TEST(MySQL_PermissionDenied) { + NYql::TIssues issues{ + NYql::TIssue( + TStringBuilder{} << MakeErrorPrefix( + "mdb.api.cloud.yandex.net:443", + "/managed-mysql/v1/clusters/etn021us5r9rhld1vgbh/hosts", + "etn021us5r9rhld1vgbh", + NYql::EDatabaseType::MySQL + ) << NoPermissionStr + ) + }; + + Test( + NYql::EDatabaseType::MySQL, + NYql::NConnector::NApi::EProtocol::NATIVE, + "https://mdb.api.cloud.yandex.net:443/managed-mysql/v1/clusters/etn021us5r9rhld1vgbh/hosts", + "403", + R"( + { + "code": 7, + "message": "Permission denied", + "details": [ + { + "@type": "type.googleapis.com/google.rpc.RequestInfo", + "requestId": "a943c092-d596-4e0e-ae7b-1f67f9d8164e" + } + ] + } + )", + NYql::TDatabaseResolverResponse::TDatabaseDescription{}, + issues + ); + } + Y_UNIT_TEST(DataStreams_PermissionDenied) { NYql::TIssues issues{ diff --git a/ydb/core/fq/libs/common/util.cpp b/ydb/core/fq/libs/common/util.cpp index 5555b364db69..61d2ea43bbda 100644 --- a/ydb/core/fq/libs/common/util.cpp +++ b/ydb/core/fq/libs/common/util.cpp @@ -126,6 +126,12 @@ TString ExtractServiceAccountId(const FederatedQuery::ConnectionSetting& setting case FederatedQuery::ConnectionSetting::kPostgresqlCluster: { return GetServiceAccountId(setting.postgresql_cluster().auth()); } + case FederatedQuery::ConnectionSetting::kGreenplumCluster: { + return GetServiceAccountId(setting.greenplum_cluster().auth()); + } + case FederatedQuery::ConnectionSetting::kMysqlCluster: { + return GetServiceAccountId(setting.mysql_cluster().auth()); + } // Do not replace with default. Adding a new connection should cause a compilation error case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET: break; @@ -157,6 +163,10 @@ TMaybe GetLogin(const FederatedQuery::ConnectionSetting& setting) { return {}; case FederatedQuery::ConnectionSetting::kPostgresqlCluster: return setting.postgresql_cluster().login(); + case FederatedQuery::ConnectionSetting::kGreenplumCluster: + return setting.greenplum_cluster().login(); + case FederatedQuery::ConnectionSetting::kMysqlCluster: + return setting.mysql_cluster().login(); } } @@ -176,6 +186,10 @@ TMaybe GetPassword(const FederatedQuery::ConnectionSetting& setting) { return {}; case FederatedQuery::ConnectionSetting::kPostgresqlCluster: return setting.postgresql_cluster().password(); + case FederatedQuery::ConnectionSetting::kGreenplumCluster: + return setting.greenplum_cluster().password(); + case FederatedQuery::ConnectionSetting::kMysqlCluster: + return setting.mysql_cluster().password(); } } @@ -195,6 +209,10 @@ EYdbComputeAuth GetYdbComputeAuthMethod(const FederatedQuery::ConnectionSetting& return GetIamAuthMethod(setting.monitoring().auth()); case FederatedQuery::ConnectionSetting::kPostgresqlCluster: return GetBasicAuthMethod(setting.postgresql_cluster().auth()); + case FederatedQuery::ConnectionSetting::kGreenplumCluster: + return GetBasicAuthMethod(setting.greenplum_cluster().auth()); + case FederatedQuery::ConnectionSetting::kMysqlCluster: + return GetBasicAuthMethod(setting.mysql_cluster().auth()); } } @@ -212,6 +230,10 @@ FederatedQuery::IamAuth GetAuth(const FederatedQuery::Connection& connection) { return connection.content().setting().monitoring().auth(); case FederatedQuery::ConnectionSetting::kPostgresqlCluster: return connection.content().setting().postgresql_cluster().auth(); + case FederatedQuery::ConnectionSetting::kGreenplumCluster: + return connection.content().setting().greenplum_cluster().auth(); + case FederatedQuery::ConnectionSetting::kMysqlCluster: + return connection.content().setting().mysql_cluster().auth(); case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET: return FederatedQuery::IamAuth{}; } diff --git a/ydb/core/fq/libs/compute/common/config.h b/ydb/core/fq/libs/compute/common/config.h index b038f8d815a7..d8a9cf63371b 100644 --- a/ydb/core/fq/libs/compute/common/config.h +++ b/ydb/core/fq/libs/compute/common/config.h @@ -164,6 +164,8 @@ class TComputeConfig { case FederatedQuery::ConnectionSetting::kObjectStorage: case FederatedQuery::ConnectionSetting::kClickhouseCluster: case FederatedQuery::ConnectionSetting::kPostgresqlCluster: + case FederatedQuery::ConnectionSetting::kGreenplumCluster: + case FederatedQuery::ConnectionSetting::kMysqlCluster: case FederatedQuery::ConnectionSetting::kYdbDatabase: return true; case FederatedQuery::ConnectionSetting::kDataStreams: diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp index 7d73bcee6855..e9aa4f877c91 100644 --- a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp +++ b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp @@ -214,8 +214,8 @@ TString MakeCreateExternalDataSourceQuery( } case FederatedQuery::ConnectionSetting::kMonitoring: break; - case FederatedQuery::ConnectionSetting::kPostgresqlCluster: - const auto schema = connectionContent.setting().postgresql_cluster().schema(); + case FederatedQuery::ConnectionSetting::kPostgresqlCluster: { + const auto pgschema = connectionContent.setting().postgresql_cluster().schema(); properties = fmt::format( R"( SOURCE_TYPE="PostgreSQL", @@ -228,7 +228,38 @@ TString MakeCreateExternalDataSourceQuery( "mdb_cluster_id"_a = EncloseAndEscapeString(connectionContent.setting().postgresql_cluster().database_id(), '"'), "database_name"_a = EncloseAndEscapeString(connectionContent.setting().postgresql_cluster().database_name(), '"'), "use_tls"_a = common.GetDisableSslForGenericDataSources() ? "false" : "true", - "schema"_a = schema ? ", SCHEMA=" + EncloseAndEscapeString(schema, '"') : TString{}); + "schema"_a = pgschema ? ", SCHEMA=" + EncloseAndEscapeString(pgschema, '"') : TString{}); + } + break; + case FederatedQuery::ConnectionSetting::kGreenplumCluster: { + const auto gpschema = connectionContent.setting().greenplum_cluster().schema(); + properties = fmt::format( + R"( + SOURCE_TYPE="Greenplum", + MDB_CLUSTER_ID={mdb_cluster_id}, + DATABASE_NAME={database_name}, + USE_TLS="{use_tls}" + {schema} + )", + "mdb_cluster_id"_a = EncloseAndEscapeString(connectionContent.setting().greenplum_cluster().database_id(), '"'), + "database_name"_a = EncloseAndEscapeString(connectionContent.setting().greenplum_cluster().database_name(), '"'), + "use_tls"_a = common.GetDisableSslForGenericDataSources() ? "false" : "true", + "schema"_a = gpschema ? ", SCHEMA=" + EncloseAndEscapeString(gpschema, '"') : TString{}); + + } + case FederatedQuery::ConnectionSetting::kMysqlCluster: { + properties = fmt::format( + R"( + SOURCE_TYPE="MySQL", + MDB_CLUSTER_ID={mdb_cluster_id}, + DATABASE_NAME={database_name}, + USE_TLS="{use_tls}" + )", + "mdb_cluster_id"_a = EncloseAndEscapeString(connectionContent.setting().mysql_cluster().database_id(), '"'), + "database_name"_a = EncloseAndEscapeString(connectionContent.setting().mysql_cluster().database_name(), '"'), + "use_tls"_a = common.GetDisableSslForGenericDataSources() ? "false" : "true"); + + } break; } diff --git a/ydb/core/fq/libs/control_plane_proxy/utils/utils.h b/ydb/core/fq/libs/control_plane_proxy/utils/utils.h index 51c5abdf8462..2eb9702387b8 100644 --- a/ydb/core/fq/libs/control_plane_proxy/utils/utils.h +++ b/ydb/core/fq/libs/control_plane_proxy/utils/utils.h @@ -31,6 +31,12 @@ TString ExtractServiceAccountIdWithConnection(const T& setting) { case FederatedQuery::ConnectionSetting::kPostgresqlCluster: { return GetServiceAccountId(setting.postgresql_cluster().auth()); } + case FederatedQuery::ConnectionSetting::kGreenplumCluster: { + return GetServiceAccountId(setting.greenplum_cluster().auth()); + } + case FederatedQuery::ConnectionSetting::kMysqlCluster: { + return GetServiceAccountId(setting.mysql_cluster().auth()); + } // Do not replace with default. Adding a new connection should cause a compilation error case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET: break; diff --git a/ydb/core/fq/libs/control_plane_storage/request_validators.cpp b/ydb/core/fq/libs/control_plane_storage/request_validators.cpp index f2016a07ddfb..996e9d7defa6 100644 --- a/ydb/core/fq/libs/control_plane_storage/request_validators.cpp +++ b/ydb/core/fq/libs/control_plane_storage/request_validators.cpp @@ -20,10 +20,15 @@ void ValidateGenericConnectionSetting( } if (!connection.database_id() && !(connection.host() && connection.port())) { - auto msg = TStringBuilder() << "content.setting.clickhouse_cluster.{database_id or host,port} field is not specified"; - issues.AddIssue( MakeErrorIssue(TIssuesIds::BAD_REQUEST,msg)); + auto msg = TStringBuilder() << "content.setting." << dataSourceKind << "_cluster.{database_id or host,port} field is not specified"; + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST,msg)); } + if (!connection.database_name()) { + auto msg = TStringBuilder() << "content.setting." << dataSourceKind << "_cluster.database_name field is not specified"; + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST,msg)); + } + if (!connection.login()) { auto msg = TStringBuilder() << "content.setting." << dataSourceKind << "_cluster.login is not specified"; issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, msg)); @@ -69,6 +74,37 @@ NYql::TIssues ValidateConnectionSetting( ValidateGenericConnectionSetting(setting.postgresql_cluster(), "postgresql", disableCurrentIam, passwordRequired, issues); break; } + case FederatedQuery::ConnectionSetting::kGreenplumCluster: { + const FederatedQuery::GreenplumCluster& greenplumCluster = setting.greenplum_cluster(); + + if (!greenplumCluster.has_auth() || greenplumCluster.auth().identity_case() == FederatedQuery::IamAuth::IDENTITY_NOT_SET) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "content.setting.greenplum_database.auth field is not specified")); + } + + if (greenplumCluster.auth().identity_case() == FederatedQuery::IamAuth::kCurrentIam && disableCurrentIam) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "current iam authorization is disabled")); + } + + if (!greenplumCluster.database_id() && !greenplumCluster.database_name()) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "content.setting.greenplum_database.{database_id or database_name} field is not specified")); + } + break; + } + case FederatedQuery::ConnectionSetting::kMysqlCluster: { + const FederatedQuery::MySQLCluster database = setting.mysql_cluster(); + if (!database.has_auth() || database.auth().identity_case() == FederatedQuery::IamAuth::IDENTITY_NOT_SET) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "content.setting.mysql_database.auth field is not specified")); + } + + if (database.auth().identity_case() == FederatedQuery::IamAuth::kCurrentIam && disableCurrentIam) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "current iam authorization is disabled")); + } + + if (!database.database_id() && !database.database_name()) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "content.setting.mysql_database.{database_id or database_name} field is not specified")); + } + break; + } case FederatedQuery::ConnectionSetting::kObjectStorage: { const FederatedQuery::ObjectStorageConnection objectStorage = setting.object_storage(); if (!objectStorage.has_auth() || objectStorage.auth().identity_case() == FederatedQuery::IamAuth::IDENTITY_NOT_SET) { diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp index 32026450cafa..7434c58aed77 100644 --- a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp +++ b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp @@ -39,6 +39,10 @@ FederatedQuery::IamAuth::IdentityCase GetIamAuth(const FederatedQuery::Connectio return setting.monitoring().auth().identity_case(); case FederatedQuery::ConnectionSetting::kPostgresqlCluster: return setting.postgresql_cluster().auth().identity_case(); + case FederatedQuery::ConnectionSetting::kGreenplumCluster: + return setting.greenplum_cluster().auth().identity_case(); + case FederatedQuery::ConnectionSetting::kMysqlCluster: + return setting.mysql_cluster().auth().identity_case(); case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET: return FederatedQuery::IamAuth::IDENTITY_NOT_SET; } diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp index ada5a7709fc7..634d835070df 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp @@ -18,6 +18,8 @@ namespace NFq { constexpr ui32 GREENPLUM_PORT = 6432; + constexpr ui32 MYSQL_PORT = 3306; + // TMdbEndpointGeneratorLegacy implements behavior required by YQL legacy ClickHouse provider class TMdbEndpointGeneratorLegacy: public NYql::IMdbEndpointGenerator { TEndpoint ToEndpoint(const NYql::IMdbEndpointGenerator::TParams& params) const override { @@ -76,13 +78,21 @@ namespace NFq { ythrow yexception() << "Unexpected protocol for PostgreSQL " << NYql::NConnector::NApi::EProtocol_Name(params.Protocol); } case NYql::EDatabaseType::Greenplum: - // https://cloud.yandex.ru/docs/managed-postgresql/operations/connect + // https://cloud.yandex.ru/docs/managed-greenplum/operations/connect switch (params.Protocol) { case NYql::NConnector::NApi::EProtocol::NATIVE: return TEndpoint(fixedHost, GREENPLUM_PORT); default: ythrow yexception() << "Unexpected protocol for Greenplum: " << NYql::NConnector::NApi::EProtocol_Name(params.Protocol); } + case NYql::EDatabaseType::MySQL: + // https://cloud.yandex.ru/docs/managed-mysql/operations/connect + switch (params.Protocol) { + case NYql::NConnector::NApi::EProtocol::NATIVE: + return TEndpoint(fixedHost, MYSQL_PORT); + default: + ythrow yexception() << "Unexpected protocol for MySQL: " << NYql::NConnector::NApi::EProtocol_Name(params.Protocol); + } default: ythrow yexception() << "Unexpected database type: " << ToString(params.DatabaseType); }; diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp index 2e27e3dcafa2..7ee10ab9683d 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp @@ -363,6 +363,7 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) { LOCATION="my-bucket", AUTH_METHOD="BASIC", LOGIN="mylogin", + DATABASE_NAME="postgres", PASSWORD_SECRET_NAME=")" << secretId << R"(" );)"; auto result = session.ExecuteSchemeQuery(query).GetValueSync(); @@ -399,6 +400,7 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) { SERVICE_ACCOUNT_ID="mysa", SERVICE_ACCOUNT_SECRET_NAME=")" << secretSaId << R"(", LOGIN="mylogin", + DATABASE_NAME="postgres", PASSWORD_SECRET_NAME=")" << secretPasswordId << R"(" );)"; auto result = session.ExecuteSchemeQuery(query).GetValueSync(); diff --git a/ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp b/ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp index 9d89bbfdf199..92faf2827c6c 100644 --- a/ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp +++ b/ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp @@ -46,6 +46,10 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) { key: "mdb_cluster_id", value: "id" } + Properties { + key: "database_name", + value: "postgres" + } } )", {NKikimrScheme::StatusAccepted}); diff --git a/ydb/public/api/protos/draft/fq.proto b/ydb/public/api/protos/draft/fq.proto index 763701ba6bd3..f047fe0acdb5 100644 --- a/ydb/public/api/protos/draft/fq.proto +++ b/ydb/public/api/protos/draft/fq.proto @@ -489,6 +489,23 @@ message PostgreSQLCluster { bool secure = 7; } +message GreenplumCluster { + string database_id = 1 [(Ydb.length).le = 1024]; + string database_name = 2 [(Ydb.length).le = 1024]; + string login = 3 [(Ydb.length).le = 1024, (Ydb.sensitive) = true]; + string password = 4 [(Ydb.length).le = 1024, (Ydb.sensitive) = true]; + string schema = 5 [(Ydb.length).le = 1024]; + IamAuth auth = 6; +} + +message MySQLCluster { + string database_id = 1 [(Ydb.length).le = 1024]; + string database_name = 2 [(Ydb.length).le = 1024]; + string login = 3 [(Ydb.length).le = 1024, (Ydb.sensitive) = true]; + string password = 4 [(Ydb.length).le = 1024, (Ydb.sensitive) = true]; + IamAuth auth = 5; +} + message ConnectionSetting { enum ConnectionType { CONNECTION_TYPE_UNSPECIFIED = 0; @@ -498,6 +515,8 @@ message ConnectionSetting { OBJECT_STORAGE = 4; MONITORING = 5; POSTGRESQL_CLUSTER = 6; + GREENPLUM_CLUSTER = 7; + MYSQL_CLUSTER = 8; } oneof connection { @@ -507,6 +526,8 @@ message ConnectionSetting { ObjectStorageConnection object_storage = 4; Monitoring monitoring = 5; PostgreSQLCluster postgresql_cluster = 6; + GreenplumCluster greenplum_cluster = 7; + MySQLCluster mysql_cluster = 8; } }