YDB FQ: set up filtering options for YQL Generic Read Actor and Lookup Actor (#10161)

vitalyisaev2 · web-flow · commit 61ddf6605c65 · 2024-10-08T11:15:31.000+03:00
diff --git a/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp b/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp
@@ -125,6 +125,7 @@ namespace NKikimr::NKqp {
             // step 3: ReadSplits
             std::vector<ui16> colData = {10, 20, 30, 40, 50};
             clientMock->ExpectReadSplits()
+                .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL)
                 .Split()
                     .Description("some binary description")
                     .Select()
@@ -221,6 +222,7 @@ namespace NKikimr::NKqp {
 
             // step 3: ReadSplits
             clientMock->ExpectReadSplits()
+                .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL)
                 .Split()
                     .Description("some binary description")
                     .Select()
@@ -313,6 +315,7 @@ namespace NKikimr::NKqp {
 
             // step 3: ReadSplits
             clientMock->ExpectReadSplits()
+                .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL)
                 .Split()
                     .Description("some binary description")
                     .Select()
@@ -418,6 +421,7 @@ namespace NKikimr::NKqp {
             std::vector<i32> filterColumnData = {42, 24};
             // clang-format off
             clientMock->ExpectReadSplits()
+                .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL)
                 .Split()
                     .Description("some binary description")
                     .Select(select)
diff --git a/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp b/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp
@@ -138,6 +138,7 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) {
 
         connectorMock->ExpectReadSplits()
             .DataSourceInstance(dsi)
+            .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY)
             .Split()
                 .Description("Actual split info is not important")
                 .Done()
diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp
@@ -168,6 +168,7 @@ namespace NYql::NDq {
 
             *readRequest.add_splits() = split;
             readRequest.Setformat(NConnector::NApi::TReadSplitsRequest_EFormat::TReadSplitsRequest_EFormat_ARROW_IPC_STREAMING);
+            readRequest.set_filtering(NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY);
             Connector->ReadSplits(readRequest).Subscribe([actorSystem = TActivationContext::ActorSystem(), selfId = SelfId()](const NConnector::TReadSplitsStreamIteratorAsyncResult& asyncResult) {
                 YQL_CLOG(DEBUG, ProviderGeneric) << "ActorId=" << selfId << " Got ReadSplitsStreamIterator from Connector";
                 auto result = ExtractFromConstFuture(asyncResult);
diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp
@@ -199,6 +199,7 @@ namespace NYql::NDq {
             // Prepare request
             NConnector::NApi::TReadSplitsRequest request;
             request.set_format(NConnector::NApi::TReadSplitsRequest::ARROW_IPC_STREAMING);
+            request.set_filtering(NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL);
             request.mutable_splits()->Reserve(Splits_.size());
 
             for (const auto& split : Splits_) {
diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h
@@ -745,6 +745,7 @@ namespace NYql::NConnector::NTest {
             DATA_SOURCE_INSTANCE_SUBBUILDER();
             SUBPROTO_BUILDER(Split, add_splits, NApi::TSplit, TSplitBuilder<TBuilder>);
             SETTER(Format, format);
+            SETTER(Filtering, filtering);
 
             TReadSplitsResultBuilder<TBuilder> Result() {
                 return TReadSplitsResultBuilder<TBuilder>(ResponseResult_, this);
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml
@@ -16,7 +16,7 @@ services:
       - /var
   fq-connector-go:
     container_name: fq-tests-ch-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
     ports:
       - 2130
     volumes:
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml
@@ -1,7 +1,7 @@
 services:
   fq-connector-go:
     container_name: fq-tests-mssql-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
     ports:
       - 2130
     volumes:
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml
@@ -1,7 +1,7 @@
 services:
   fq-connector-go:
     container_name: fq-tests-mysql-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
     ports:
       - 2130
     volumes:
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml
@@ -1,7 +1,7 @@
 services:
   fq-connector-go:
     container_name: fq-tests-oracle-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
     ports:
       - 2130
     volumes:
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml
@@ -1,7 +1,7 @@
 services:
   fq-connector-go:
     container_name: fq-tests-pg-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
     ports:
       - 2130
     volumes:
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml
@@ -8,7 +8,7 @@ services:
         dnsmasq;
         /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml"
     container_name: fq-tests-ydb-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
     ports:
       - 2130
     volumes:
diff --git a/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml
@@ -12,7 +12,7 @@ services:
       - 8123
   fq-connector-go:
     container_name: fq-tests-join-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
     ports:
       - 2130
     volumes:
diff --git a/ydb/tests/fq/generic/analytics/docker-compose.yml b/ydb/tests/fq/generic/analytics/docker-compose.yml
@@ -19,7 +19,7 @@ services:
         echo \"$$(dig tests-fq-generic-analytics-ydb +short) tests-fq-generic-analytics-ydb\" >> /etc/hosts; cat /etc/hosts;
         /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml"
     container_name: tests-fq-generic-analytics-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
     ports:
       - "2130"
   greenplum:
diff --git a/ydb/tests/fq/generic/streaming/docker-compose.yml b/ydb/tests/fq/generic/streaming/docker-compose.yml
@@ -5,7 +5,7 @@ services:
         echo \"$$(dig tests-fq-generic-streaming-ydb +short) tests-fq-generic-streaming-ydb\" >> /etc/hosts; cat /etc/hosts;
         /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml"
     container_name: tests-fq-generic-streaming-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
     ports:
       - "2130"
   ydb:
