Skip to content

Commit

Permalink
YDB FQ: avoid outdated syntax "SELECT * FROM cluster.db.table" (copy … (
Browse files Browse the repository at this point in the history 
ydb-platform#6970)
  • Loading branch information
@uzhastik
uzhastik committed Sep 12, 2024
1 parent 2f4cb25 commit f16771e
Show file tree
Hide file tree
Showing 9 changed files with 38 additions and 98 deletions.
2 changes: 2 additions & 0 deletions ydb/library/yql/providers/generic/connector/libcpp/error.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ namespace NYql::NConnector {
return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_UNSUPPORTED;
case ::Ydb::StatusIds::StatusCode::StatusIds_StatusCode_NOT_FOUND:
return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_BAD_REQUEST;
case ::Ydb::StatusIds::StatusCode::StatusIds_StatusCode_SCHEME_ERROR:
return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_SCHEME_ERROR;
default:
ythrow yexception() << "Unexpected YDB status code: " << ::Ydb::StatusIds::StatusCode_Name(error.status());
}
Expand Down
2 changes: 1 addition & 1 deletion ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ services:
- 8123
fq-connector-go:
container_name: fq-tests-ch-fq-connector-go
image: ghcr.io/ydb-platform/fq-connector-go:v0.4.9@sha256:3a1fe086be50c0edbae2c2b284aee5ce76bd056d7f46cb460919ec37a6f8ab5c
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
ports:
- 2130
volumes:
Expand Down
2 changes: 1 addition & 1 deletion ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
services:
fq-connector-go:
container_name: fq-tests-pg-fq-connector-go
image: ghcr.io/ydb-platform/fq-connector-go:v0.4.9@sha256:3a1fe086be50c0edbae2c2b284aee5ce76bd056d7f46cb460919ec37a6f8ab5c
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
ports:
- 2130
volumes:
Expand Down
2 changes: 1 addition & 1 deletion ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ services:
echo \"$$(dig fq-tests-ydb-ydb +short) fq-tests-ydb-ydb\" >> /etc/hosts; cat /etc/hosts;
/opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml"
container_name: fq-tests-ydb-fq-connector-go
image: ghcr.io/ydb-platform/fq-connector-go:v0.4.9@sha256:3a1fe086be50c0edbae2c2b284aee5ce76bd056d7f46cb460919ec37a6f8ab5c
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
ports:
- 2130
volumes:
Expand Down
2 changes: 1 addition & 1 deletion ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ services:
- 8123
fq-connector-go:
container_name: fq-tests-join-fq-connector-go
image: ghcr.io/ydb-platform/fq-connector-go:v0.4.9@sha256:3a1fe086be50c0edbae2c2b284aee5ce76bd056d7f46cb460919ec37a6f8ab5c
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
ports:
- 2130
volumes:
Expand Down
36 changes: 27 additions & 9 deletions ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,14 +107,12 @@ namespace NYql {
NYql::TGenericClusterConfig& clusterConfig) {
auto it = properties.find("database_name");
if (it == properties.cend()) {
// TODO: make this property required during https://st.yandex-team.ru/YQ-2494
// ythrow yexception() << "missing 'DATABASE_NAME' value";
// DATABASE_NAME is a mandatory field for the most of databases,
// however, managed YDB does not require it, so we have to accept empty values here.
return;
}

if (!it->second) {
// TODO: make this property required during https://st.yandex-team.ru/YQ-2494
// ythrow yexception() << "invalid 'DATABASE_NAME' value: '" << it->second << "'";
return;
}

Expand All @@ -125,14 +123,12 @@ namespace NYql {
NYql::TGenericClusterConfig& clusterConfig) {
auto it = properties.find("schema");
if (it == properties.cend()) {
// TODO: make this property required during https://st.yandex-team.ru/YQ-2494
// ythrow yexception() << "missing 'SCHEMA' value";
// SCHEMA is optional field
return;
}

if (!it->second) {
// TODO: make this property required during https://st.yandex-team.ru/YQ-2494
// ythrow yexception() << "invalid 'SCHEMA' value: '" << it->second << "'";
// SCHEMA is optional field
return;
}

Expand Down Expand Up @@ -318,9 +314,20 @@ namespace NYql {
}

static const TSet<NConnector::NApi::EDataSourceKind> managedDatabaseKinds{
NConnector::NApi::EDataSourceKind::CLICKHOUSE,
NConnector::NApi::EDataSourceKind::GREENPLUM,
NConnector::NApi::EDataSourceKind::MYSQL,
NConnector::NApi::EDataSourceKind::POSTGRESQL,
NConnector::NApi::EDataSourceKind::YDB,
};

static const TSet<NConnector::NApi::EDataSourceKind> traditionalRelationalDatabaseKinds{
NConnector::NApi::EDataSourceKind::CLICKHOUSE,
NConnector::NApi::EDataSourceKind::YDB};
NConnector::NApi::EDataSourceKind::GREENPLUM,
NConnector::NApi::EDataSourceKind::MS_SQL_SERVER,
NConnector::NApi::EDataSourceKind::MYSQL,
NConnector::NApi::EDataSourceKind::POSTGRESQL,
};

void ValidateGenericClusterConfig(
const NYql::TGenericClusterConfig& clusterConfig,
Expand Down Expand Up @@ -396,6 +403,17 @@ namespace NYql {
}
}

// All the databases with exception to managed YDB:
// * DATABASE_NAME is mandatory field
if (traditionalRelationalDatabaseKinds.contains(clusterConfig.GetKind())) {
if (!clusterConfig.GetDatabaseName()) {
return ValidationError(
clusterConfig,
context,
"You must provide database name explicitly");
}
}

// check required fields
if (!clusterConfig.GetName()) {
return ValidationError(clusterConfig, context, "empty field 'Name'");
Expand Down
54 changes: 2 additions & 52 deletions ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,8 +121,6 @@ namespace NYql {
const auto& clusterConfig = State_->Configuration->ClusterNamesToClusterConfigs[clusterName];
const auto& endpoint = clusterConfig.endpoint();

Generic::TSource source;

YQL_CLOG(INFO, ProviderGeneric)
<< "Filling source settings"
<< ": cluster: " << clusterName
Expand All @@ -137,8 +135,9 @@ namespace NYql {
}

// prepare select
Generic::TSource source;
auto select = source.mutable_select();
select->mutable_from()->set_table(TString(table));
select->mutable_from()->set_table(table);
select->mutable_data_source_instance()->CopyFrom(tableMeta.value()->DataSourceInstance);

auto items = select->mutable_what()->mutable_items();
Expand Down Expand Up @@ -254,55 +253,6 @@ namespace NYql {
void RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) override {
RegisterDqGenericMkqlCompilers(compiler, State_);
}

void FillLookupSourceSettings(const TExprNode& node, ::google::protobuf::Any& protoSettings, TString& sourceType) override {
const TDqLookupSourceWrap wrap(&node);
const auto settings = wrap.Input().Cast<TGenSourceSettings>();

const auto& clusterName = wrap.DataSource().Cast<TGenDataSource>().Cluster().StringValue();
const auto& table = settings.Table().StringValue();
const auto& clusterConfig = State_->Configuration->ClusterNamesToClusterConfigs[clusterName];
const auto& endpoint = clusterConfig.endpoint();

// for backward compability full path can be used (cluster_name.`db_name.table`)
// TODO: simplify during https://st.yandex-team.ru/YQ-2494
TStringBuf db, dbTable;
if (!TStringBuf(table).TrySplit('.', db, dbTable)) {
dbTable = table;
}

YQL_CLOG(INFO, ProviderGeneric)
<< "Filling lookup source settings"
<< ": cluster: " << clusterName
<< ", table: " << table
<< ", endpoint: " << endpoint.ShortDebugString();

auto [tableMeta, issue] = State_->GetTable(clusterName, table);
if (issue.has_value()) {
ythrow yexception() << "Get table metadata: " << issue.value();
}

Generic::TLookupSource source;
source.set_table(TString(dbTable));
*source.mutable_data_source_instance() = tableMeta.value()->DataSourceInstance;

// Managed YDB supports access via IAM token.
// If exist, copy service account creds to obtain tokens during request execution phase.
// If exists, copy previously created token.
if (clusterConfig.kind() == NConnector::NApi::EDataSourceKind::YDB) {
source.SetServiceAccountId(clusterConfig.GetServiceAccountId());
source.SetServiceAccountIdSignature(clusterConfig.GetServiceAccountIdSignature());
source.SetToken(State_->Types->Credentials->FindCredentialContent(
"default_" + clusterConfig.name(),
"default_generic",
clusterConfig.GetToken()));
}

// preserve source description for read actor
protoSettings.PackFrom(source);
sourceType = GetSourceType(source.data_source_instance());
}

private:
const TGenericState::TPtr State_;
};
Expand Down
34 changes: 2 additions & 32 deletions ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,38 +367,8 @@ namespace NYql {

void FillTablePath(NConnector::NApi::TDescribeTableRequest& request, const TGenericClusterConfig& clusterConfig,
const TString& tablePath) {
// for backward compability full path can be used (cluster_name.`db_name.table`)
// TODO: simplify during https://st.yandex-team.ru/YQ-2494
const auto dataSourceKind = clusterConfig.GetKind();
const auto& dbNameFromConfig = clusterConfig.GetDatabaseName();
TStringBuf dbNameTarget, tableName;
auto isFullPath = TStringBuf(tablePath).TrySplit('.', dbNameTarget, tableName);

if (!dbNameFromConfig.empty()) {
dbNameTarget = dbNameFromConfig;
if (!isFullPath) {
tableName = tablePath;
}
} else if (!isFullPath) {
tableName = tablePath;
switch (dataSourceKind) {
case NYql::NConnector::NApi::CLICKHOUSE:
dbNameTarget = "default";
break;
case NYql::NConnector::NApi::POSTGRESQL:
dbNameTarget = "postgres";
break;
case NYql::NConnector::NApi::MS_SQL_SERVER:
dbNameTarget = "mssqlserver";
break;
default:
ythrow yexception() << "You must provide database name explicitly for data source kind: '"
<< NYql::NConnector::NApi::EDataSourceKind_Name(dataSourceKind) << "'";
}
} // else take database name from table path

request.mutable_data_source_instance()->set_database(TString(dbNameTarget));
request.set_table(TString(tableName));
request.mutable_data_source_instance()->set_database(clusterConfig.GetDatabaseName());
request.set_table(tablePath);
}

private:
Expand Down
2 changes: 1 addition & 1 deletion ydb/tests/fq/generic/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ services:
echo \"$$(dig tests-fq-generic-ydb +short) tests-fq-generic-ydb\" >> /etc/hosts; cat /etc/hosts;
/opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml"
container_name: tests-fq-generic-fq-connector-go
image: ghcr.io/ydb-platform/fq-connector-go:v0.4.9@sha256:3a1fe086be50c0edbae2c2b284aee5ce76bd056d7f46cb460919ec37a6f8ab5c
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
ports:
- "2130"
postgresql:
Expand Down

0 comments on commit f16771e

Please sign in to comment.