From 0da9d0ec96aaf365b31be52d9e0223e9102227ef Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Tue, 8 Oct 2024 11:15:31 +0300 Subject: [PATCH] YDB FQ: set up filtering options for YQL Generic Read Actor and Lookup Actor (#10161) --- .../ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp | 4 ++++ .../generic/actors/ut/yql_generic_lookup_actor_ut.cpp | 1 + .../yql/providers/generic/actors/yql_generic_lookup_actor.cpp | 1 + .../yql/providers/generic/actors/yql_generic_read_actor.cpp | 1 + .../connector/libcpp/ut_helpers/connector_client_mock.h | 1 + .../connector/tests/datasource/clickhouse/docker-compose.yml | 2 +- .../connector/tests/datasource/postgresql/docker-compose.yml | 2 +- .../generic/connector/tests/datasource/ydb/docker-compose.yml | 2 +- .../providers/generic/connector/tests/join/docker-compose.yml | 2 +- 9 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp b/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp index 175718089600..c461fe68dfac 100644 --- a/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp @@ -125,6 +125,7 @@ namespace NKikimr::NKqp { // step 3: ReadSplits std::vector colData = {10, 20, 30, 40, 50}; clientMock->ExpectReadSplits() + .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL) .Split() .Description("some binary description") .Select() @@ -221,6 +222,7 @@ namespace NKikimr::NKqp { // step 3: ReadSplits clientMock->ExpectReadSplits() + .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL) .Split() .Description("some binary description") .Select() @@ -313,6 +315,7 @@ namespace NKikimr::NKqp { // step 3: ReadSplits clientMock->ExpectReadSplits() + .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL) .Split() .Description("some binary description") .Select() @@ -418,6 +421,7 @@ namespace NKikimr::NKqp { std::vector filterColumnData = {42, 24}; // clang-format off clientMock->ExpectReadSplits() + .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL) .Split() .Description("some binary description") .Select(select) diff --git a/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp b/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp index 48c5b2951098..d7b77b8bbda6 100644 --- a/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp +++ b/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp @@ -128,6 +128,7 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) { connectorMock->ExpectReadSplits() .DataSourceInstance(dsi) + .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY) .Split() .Description("Actual split info is not important") .Done() diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp index 18374715fa32..21090f61e5b8 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp @@ -162,6 +162,7 @@ namespace NYql::NDq { *readRequest.add_splits() = split; readRequest.Setformat(NConnector::NApi::TReadSplitsRequest_EFormat::TReadSplitsRequest_EFormat_ARROW_IPC_STREAMING); + readRequest.set_filtering(NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY); Connector->ReadSplits(readRequest).Subscribe([actorSystem = TActivationContext::ActorSystem(), selfId = SelfId()](const NConnector::TReadSplitsStreamIteratorAsyncResult& asyncResult) { YQL_CLOG(DEBUG, ProviderGeneric) << "ActorId=" << selfId << " Got ReadSplitsStreamIterator from Connector"; auto result = ExtractFromConstFuture(asyncResult); diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp index 93b3ba116adb..4c664f665826 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp @@ -199,6 +199,7 @@ namespace NYql::NDq { // Prepare request NConnector::NApi::TReadSplitsRequest request; request.set_format(NConnector::NApi::TReadSplitsRequest::ARROW_IPC_STREAMING); + request.set_filtering(NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL); request.mutable_splits()->Reserve(Splits_.size()); for (const auto& split : Splits_) { diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h index ae92f1c58631..7f764611593a 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h @@ -745,6 +745,7 @@ namespace NYql::NConnector::NTest { DATA_SOURCE_INSTANCE_SUBBUILDER(); SUBPROTO_BUILDER(Split, add_splits, NApi::TSplit, TSplitBuilder); SETTER(Format, format); + SETTER(Filtering, filtering); TReadSplitsResultBuilder Result() { return TReadSplitsResultBuilder(ResponseResult_, this); diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml index c8fb08a8927f..d79b229395e3 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml @@ -12,7 +12,7 @@ services: - 8123 fq-connector-go: container_name: fq-tests-ch-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.7-rc.1@sha256:f12475f346105d7bc630e7b85f51dce980bf9833f6ce0625c6f1191b1a1de923 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml index 5069925061a1..d7a98b01993c 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-pg-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.7-rc.1@sha256:f12475f346105d7bc630e7b85f51dce980bf9833f6ce0625c6f1191b1a1de923 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml index baf6c9e4ab3e..6282e6653185 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml @@ -5,7 +5,7 @@ services: echo \"$$(dig fq-tests-ydb-ydb +short) fq-tests-ydb-ydb\" >> /etc/hosts; cat /etc/hosts; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: fq-tests-ydb-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.7-rc.1@sha256:f12475f346105d7bc630e7b85f51dce980bf9833f6ce0625c6f1191b1a1de923 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml index daf28b70f920..6d7996d7e149 100644 --- a/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml @@ -12,7 +12,7 @@ services: - 8123 fq-connector-go: container_name: fq-tests-join-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.7-rc.1@sha256:f12475f346105d7bc630e7b85f51dce980bf9833f6ce0625c6f1191b1a1de923 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: