Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merge #9778 #10161 #10202

Merged
merged 2 commits into from
Oct 8, 2024
Merged

Merge #9778 #10161 #10202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c14cf09
YDB FQ: Connector filtering modes (#9778)
vitalyisaev2 Sep 27, 2024
0da9d0e
YDB FQ: set up filtering options for YQL Generic Read Actor and Looku…
vitalyisaev2 Oct 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ namespace NKikimr::NKqp {
// step 3: ReadSplits
std::vector<ui16> colData = {10, 20, 30, 40, 50};
clientMock->ExpectReadSplits()
.Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL)
.Split()
.Description("some binary description")
.Select()
Expand Down Expand Up @@ -221,6 +222,7 @@ namespace NKikimr::NKqp {

// step 3: ReadSplits
clientMock->ExpectReadSplits()
.Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL)
.Split()
.Description("some binary description")
.Select()
Expand Down Expand Up @@ -313,6 +315,7 @@ namespace NKikimr::NKqp {

// step 3: ReadSplits
clientMock->ExpectReadSplits()
.Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL)
.Split()
.Description("some binary description")
.Select()
Expand Down Expand Up @@ -418,6 +421,7 @@ namespace NKikimr::NKqp {
std::vector<i32> filterColumnData = {42, 24};
// clang-format off
clientMock->ExpectReadSplits()
.Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL)
.Split()
.Description("some binary description")
.Select(select)
Expand Down
1 change: 1 addition & 0 deletions ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) {

connectorMock->ExpectReadSplits()
.DataSourceInstance(dsi)
.Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY)
.Split()
.Description("Actual split info is not important")
.Done()
Expand Down
1 change: 1 addition & 0 deletions ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ namespace NYql::NDq {

*readRequest.add_splits() = split;
readRequest.Setformat(NConnector::NApi::TReadSplitsRequest_EFormat::TReadSplitsRequest_EFormat_ARROW_IPC_STREAMING);
readRequest.set_filtering(NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY);
Connector->ReadSplits(readRequest).Subscribe([actorSystem = TActivationContext::ActorSystem(), selfId = SelfId()](const NConnector::TReadSplitsStreamIteratorAsyncResult& asyncResult) {
YQL_CLOG(DEBUG, ProviderGeneric) << "ActorId=" << selfId << " Got ReadSplitsStreamIterator from Connector";
auto result = ExtractFromConstFuture(asyncResult);
Expand Down
1 change: 1 addition & 0 deletions ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ namespace NYql::NDq {
// Prepare request
NConnector::NApi::TReadSplitsRequest request;
request.set_format(NConnector::NApi::TReadSplitsRequest::ARROW_IPC_STREAMING);
request.set_filtering(NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL);
request.mutable_splits()->Reserve(Splits_.size());

for (const auto& split : Splits_) {
Expand Down
16 changes: 16 additions & 0 deletions ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,22 @@ message TReadSplitsRequest {
// If empty, the connector will return the split data from the very beginning.
TContinuation continuation = 6;

enum EFiltering {
FILTERING_UNSPECIFIED = 0;
// If Connector cannot push down the predicate to the data source completely
// (due to the lack of data type support, for example), it doesn't apply filter at all
// and returns the full result of `SELECT columns FROM table` (no WHERE clause).
// It's YDB's duty to filter the output on its own side.
FILTERING_OPTIONAL = 1;
// If Connector cannot push down the predicate to the data source completely,
// it terminates the request and returns an error.
FILTERING_MANDATORY = 2;
}

// Determines various modes of server behavior in the context of predicate pushdown.
// If not set, the default value is `FILTERING_OPTIONAL`.
EFiltering filtering = 7;

reserved 5;
}

Expand Down
1 change: 1 addition & 0 deletions ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -745,6 +745,7 @@ namespace NYql::NConnector::NTest {
DATA_SOURCE_INSTANCE_SUBBUILDER();
SUBPROTO_BUILDER(Split, add_splits, NApi::TSplit, TSplitBuilder<TBuilder>);
SETTER(Format, format);
SETTER(Filtering, filtering);

TReadSplitsResultBuilder<TBuilder> Result() {
return TReadSplitsResultBuilder<TBuilder>(ResponseResult_, this);
Expand Down
2 changes: 1 addition & 1 deletion ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ services:
- 8123
fq-connector-go:
container_name: fq-tests-ch-fq-connector-go
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.7-rc.1@sha256:f12475f346105d7bc630e7b85f51dce980bf9833f6ce0625c6f1191b1a1de923
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
ports:
- 2130
volumes:
Expand Down
2 changes: 1 addition & 1 deletion ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
services:
fq-connector-go:
container_name: fq-tests-pg-fq-connector-go
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.7-rc.1@sha256:f12475f346105d7bc630e7b85f51dce980bf9833f6ce0625c6f1191b1a1de923
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
ports:
- 2130
volumes:
Expand Down
2 changes: 1 addition & 1 deletion ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ services:
echo \"$$(dig fq-tests-ydb-ydb +short) fq-tests-ydb-ydb\" >> /etc/hosts; cat /etc/hosts;
/opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml"
container_name: fq-tests-ydb-fq-connector-go
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.7-rc.1@sha256:f12475f346105d7bc630e7b85f51dce980bf9833f6ce0625c6f1191b1a1de923
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
ports:
- 2130
volumes:
Expand Down
2 changes: 1 addition & 1 deletion ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ services:
- 8123
fq-connector-go:
container_name: fq-tests-join-fq-connector-go
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.7-rc.1@sha256:f12475f346105d7bc630e7b85f51dce980bf9833f6ce0625c6f1191b1a1de923
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
ports:
- 2130
volumes:
Expand Down
Loading