From a57e612ff6cc192e4d5f94db0c0273ee78af82b4 Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Mon, 7 Oct 2024 11:05:07 +0000 Subject: [PATCH 1/6] YDB FQ: set up filtering options for YQL Generic Read Actor and Lookup Actor --- .../yql/providers/generic/actors/yql_generic_lookup_actor.cpp | 1 + .../yql/providers/generic/actors/yql_generic_read_actor.cpp | 1 + .../connector/tests/datasource/clickhouse/docker-compose.yml | 2 +- .../connector/tests/datasource/ms_sql_server/docker-compose.yml | 2 +- .../generic/connector/tests/datasource/mysql/docker-compose.yml | 2 +- .../connector/tests/datasource/oracle/docker-compose.yml | 2 +- .../connector/tests/datasource/postgresql/docker-compose.yml | 2 +- .../generic/connector/tests/datasource/ydb/docker-compose.yml | 2 +- .../providers/generic/connector/tests/join/docker-compose.yml | 2 +- ydb/tests/fq/generic/analytics/docker-compose.yml | 2 +- ydb/tests/fq/generic/streaming/docker-compose.yml | 2 +- 11 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp index eba5f8a1f207..90db928a8f2b 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp @@ -168,6 +168,7 @@ namespace NYql::NDq { *readRequest.add_splits() = split; readRequest.Setformat(NConnector::NApi::TReadSplitsRequest_EFormat::TReadSplitsRequest_EFormat_ARROW_IPC_STREAMING); + readRequest.set_filtering(::NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY); Connector->ReadSplits(readRequest).Subscribe([actorSystem = TActivationContext::ActorSystem(), selfId = SelfId()](const NConnector::TReadSplitsStreamIteratorAsyncResult& asyncResult) { YQL_CLOG(DEBUG, ProviderGeneric) << "ActorId=" << selfId << " Got ReadSplitsStreamIterator from Connector"; auto result = ExtractFromConstFuture(asyncResult); diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp index c74f7a9b4b4a..031eac3a069b 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp @@ -199,6 +199,7 @@ namespace NYql::NDq { // Prepare request NConnector::NApi::TReadSplitsRequest request; request.set_format(NConnector::NApi::TReadSplitsRequest::ARROW_IPC_STREAMING); + request.set_filtering(::NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL); request.mutable_splits()->Reserve(Splits_.size()); for (const auto& split : Splits_) { diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml index a459f1239505..1ae3fd60aa54 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml @@ -16,7 +16,7 @@ services: - /var fq-connector-go: container_name: fq-tests-ch-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml index b65e7c70dc06..43f328927139 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-mssql-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml index 8ba878edf6b3..4971bacac70b 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-mysql-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml index a2ecbd99242d..f4a815b74448 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-oracle-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml index 6d64be978bc7..53562fc7fd1c 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-pg-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml index b97b90b21151..b2922cb72451 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml @@ -8,7 +8,7 @@ services: dnsmasq; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: fq-tests-ydb-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml index 6f8acd4ace53..992d594daaf6 100644 --- a/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml @@ -12,7 +12,7 @@ services: - 8123 fq-connector-go: container_name: fq-tests-join-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/tests/fq/generic/analytics/docker-compose.yml b/ydb/tests/fq/generic/analytics/docker-compose.yml index ceb5ea309884..f702857495a2 100644 --- a/ydb/tests/fq/generic/analytics/docker-compose.yml +++ b/ydb/tests/fq/generic/analytics/docker-compose.yml @@ -19,7 +19,7 @@ services: echo \"$$(dig tests-fq-generic-analytics-ydb +short) tests-fq-generic-analytics-ydb\" >> /etc/hosts; cat /etc/hosts; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: tests-fq-generic-analytics-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - "2130" greenplum: diff --git a/ydb/tests/fq/generic/streaming/docker-compose.yml b/ydb/tests/fq/generic/streaming/docker-compose.yml index c77865cccb98..2108a4b67504 100644 --- a/ydb/tests/fq/generic/streaming/docker-compose.yml +++ b/ydb/tests/fq/generic/streaming/docker-compose.yml @@ -5,7 +5,7 @@ services: echo \"$$(dig tests-fq-generic-streaming-ydb +short) tests-fq-generic-streaming-ydb\" >> /etc/hosts; cat /etc/hosts; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: tests-fq-generic-streaming-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - "2130" ydb: From 9d9d0abf33b4c86f4c754d3c81f47c2faaa196f6 Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Mon, 7 Oct 2024 11:09:12 +0000 Subject: [PATCH 2/6] Minor changes --- .../yql/providers/generic/actors/yql_generic_read_actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp index 031eac3a069b..f3b32eb2d27a 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp @@ -199,7 +199,7 @@ namespace NYql::NDq { // Prepare request NConnector::NApi::TReadSplitsRequest request; request.set_format(NConnector::NApi::TReadSplitsRequest::ARROW_IPC_STREAMING); - request.set_filtering(::NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL); + request.set_filtering(NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL); request.mutable_splits()->Reserve(Splits_.size()); for (const auto& split : Splits_) { From 12bae762271b454165484b98b26c45cb9268a0de Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Mon, 7 Oct 2024 11:18:00 +0000 Subject: [PATCH 3/6] Minor changes --- .../yql/providers/generic/actors/yql_generic_lookup_actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp index 90db928a8f2b..25c809c63d79 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp @@ -168,7 +168,7 @@ namespace NYql::NDq { *readRequest.add_splits() = split; readRequest.Setformat(NConnector::NApi::TReadSplitsRequest_EFormat::TReadSplitsRequest_EFormat_ARROW_IPC_STREAMING); - readRequest.set_filtering(::NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY); + readRequest.set_filtering(NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY); Connector->ReadSplits(readRequest).Subscribe([actorSystem = TActivationContext::ActorSystem(), selfId = SelfId()](const NConnector::TReadSplitsStreamIteratorAsyncResult& asyncResult) { YQL_CLOG(DEBUG, ProviderGeneric) << "ActorId=" << selfId << " Got ReadSplitsStreamIterator from Connector"; auto result = ExtractFromConstFuture(asyncResult); From f4d9c7f94e3741359fdc3744180dbe317acf78d5 Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Mon, 7 Oct 2024 14:11:43 +0000 Subject: [PATCH 4/6] Fix ut --- .../generic/connector/libcpp/ut_helpers/connector_client_mock.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h index ae92f1c58631..4623afd4a6c5 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h @@ -745,6 +745,7 @@ namespace NYql::NConnector::NTest { DATA_SOURCE_INSTANCE_SUBBUILDER(); SUBPROTO_BUILDER(Split, add_splits, NApi::TSplit, TSplitBuilder); SETTER(Format, format); + SETTER(Filtering, filtering); TReadSplitsResultBuilder Result() { return TReadSplitsResultBuilder(ResponseResult_, this); @@ -752,6 +753,7 @@ namespace NYql::NConnector::NTest { void FillWithDefaults() { Format(NApi::TReadSplitsRequest::ARROW_IPC_STREAMING); + Filtering(NApi::TReadSplitsRequest::FILTERING_OPTIONAL); } private: From 6680f74417892cd59d8a0fa9e1115d5a546d82dd Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Mon, 7 Oct 2024 16:11:01 +0000 Subject: [PATCH 5/6] Fix ut --- .../ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp | 4 ++++ .../generic/actors/ut/yql_generic_lookup_actor_ut.cpp | 1 + 2 files changed, 5 insertions(+) diff --git a/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp b/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp index 175718089600..c461fe68dfac 100644 --- a/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp @@ -125,6 +125,7 @@ namespace NKikimr::NKqp { // step 3: ReadSplits std::vector colData = {10, 20, 30, 40, 50}; clientMock->ExpectReadSplits() + .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL) .Split() .Description("some binary description") .Select() @@ -221,6 +222,7 @@ namespace NKikimr::NKqp { // step 3: ReadSplits clientMock->ExpectReadSplits() + .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL) .Split() .Description("some binary description") .Select() @@ -313,6 +315,7 @@ namespace NKikimr::NKqp { // step 3: ReadSplits clientMock->ExpectReadSplits() + .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL) .Split() .Description("some binary description") .Select() @@ -418,6 +421,7 @@ namespace NKikimr::NKqp { std::vector filterColumnData = {42, 24}; // clang-format off clientMock->ExpectReadSplits() + .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL) .Split() .Description("some binary description") .Select(select) diff --git a/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp b/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp index cc4bda4bd69a..e85bdfa01e14 100644 --- a/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp +++ b/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp @@ -138,6 +138,7 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) { connectorMock->ExpectReadSplits() .DataSourceInstance(dsi) + .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY) .Split() .Description("Actual split info is not important") .Done() From 0849b862d2c31ce631c02998ebadac6bea09f480 Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Mon, 7 Oct 2024 16:15:45 +0000 Subject: [PATCH 6/6] Fix ut --- .../generic/connector/libcpp/ut_helpers/connector_client_mock.h | 1 - 1 file changed, 1 deletion(-) diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h index 4623afd4a6c5..7f764611593a 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h @@ -753,7 +753,6 @@ namespace NYql::NConnector::NTest { void FillWithDefaults() { Format(NApi::TReadSplitsRequest::ARROW_IPC_STREAMING); - Filtering(NApi::TReadSplitsRequest::FILTERING_OPTIONAL); } private: