Skip to content

Commit 8731e32

Browse files
vitalyisaev2vitalyisaev2
authored
2 parents 5c06dfa + ef45b55 commit 8731e32

11 files changed

+75
-36
lines changed

‎ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) {
160160
;
161161
// clang-format on
162162

163-
NYql::NGeneric::TLookupSource lookupSourceSettings;
163+
NYql::Generic::TLookupSource lookupSourceSettings;
164164
*lookupSourceSettings.mutable_data_source_instance() = dsi;
165165
lookupSourceSettings.Settable("lookup_test");
166166
lookupSourceSettings.SetServiceAccountId("testsaid");
@@ -351,7 +351,7 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) {
351351
}
352352
// clang-format on
353353

354-
NYql::NGeneric::TLookupSource lookupSourceSettings;
354+
NYql::Generic::TLookupSource lookupSourceSettings;
355355
*lookupSourceSettings.mutable_data_source_instance() = dsi;
356356
lookupSourceSettings.Settable("lookup_test");
357357
lookupSourceSettings.SetServiceAccountId("testsaid");

‎ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ namespace NYql::NDq {
6868
::NMonitoring::TDynamicCounterPtr taskCounters,
6969
std::shared_ptr<NKikimr::NMiniKQL::TScopedAlloc> alloc,
7070
std::shared_ptr<IDqAsyncLookupSource::TKeyTypeHelper> keyTypeHelper,
71-
NYql::NGeneric::TLookupSource&& lookupSource,
71+
Generic::TLookupSource&& lookupSource,
7272
const NKikimr::NMiniKQL::TStructType* keyType,
7373
const NKikimr::NMiniKQL::TStructType* payloadType,
7474
const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,
@@ -499,7 +499,7 @@ namespace NYql::NDq {
499499
const NActors::TActorId ParentId;
500500
std::shared_ptr<NKikimr::NMiniKQL::TScopedAlloc> Alloc;
501501
std::shared_ptr<TKeyTypeHelper> KeyTypeHelper;
502-
const NYql::NGeneric::TLookupSource LookupSource;
502+
const Generic::TLookupSource LookupSource;
503503
const NKikimr::NMiniKQL::TStructType* const KeyType;
504504
const NKikimr::NMiniKQL::TStructType* const PayloadType;
505505
const NKikimr::NMiniKQL::TStructType* const SelectResultType; // columns from KeyType + PayloadType
@@ -528,7 +528,7 @@ namespace NYql::NDq {
528528
::NMonitoring::TDynamicCounterPtr taskCounters,
529529
std::shared_ptr<NKikimr::NMiniKQL::TScopedAlloc> alloc,
530530
std::shared_ptr<IDqAsyncLookupSource::TKeyTypeHelper> keyTypeHelper,
531-
NYql::NGeneric::TLookupSource&& lookupSource,
531+
Generic::TLookupSource&& lookupSource,
532532
const NKikimr::NMiniKQL::TStructType* keyType,
533533
const NKikimr::NMiniKQL::TStructType* payloadType,
534534
const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,

‎ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ namespace NYql::NDq {
1818
::NMonitoring::TDynamicCounterPtr taskCounters,
1919
std::shared_ptr<NKikimr::NMiniKQL::TScopedAlloc> alloc,
2020
std::shared_ptr<IDqAsyncLookupSource::TKeyTypeHelper> keyTypeHelper,
21-
NYql::NGeneric::TLookupSource&& lookupSource,
21+
Generic::TLookupSource&& lookupSource,
2222
const NKikimr::NMiniKQL::TStructType* keyType,
2323
const NKikimr::NMiniKQL::TStructType* payloadType,
2424
const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv,

‎ydb/library/yql/providers/generic/actors/yql_generic_provider_factories.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ namespace NYql::NDq {
1111
ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
1212
NYql::NConnector::IClient::TPtr genericClient) {
1313
auto readActorFactory = [credentialsFactory, genericClient](
14-
NGeneric::TSource&& settings,
14+
Generic::TSource&& settings,
1515
IDqAsyncIoFactory::TSourceArguments&& args) {
1616
return CreateGenericReadActor(
1717
genericClient,
@@ -27,7 +27,7 @@ namespace NYql::NDq {
2727
args.HolderFactory);
2828
};
2929

30-
auto lookupActorFactory = [credentialsFactory, genericClient](NYql::NGeneric::TLookupSource&& lookupSource, IDqAsyncIoFactory::TLookupSourceArguments&& args) {
30+
auto lookupActorFactory = [credentialsFactory, genericClient](Generic::TLookupSource&& lookupSource, IDqAsyncIoFactory::TLookupSourceArguments&& args) {
3131
return CreateGenericLookupActor(
3232
genericClient,
3333
credentialsFactory,
@@ -52,8 +52,8 @@ namespace NYql::NDq {
5252
"MsSQLServerGeneric",
5353
"OracleGeneric",
5454
"LoggingGeneric"}) {
55-
factory.RegisterSource<NGeneric::TSource>(name, readActorFactory);
56-
factory.RegisterLookupSource<NGeneric::TLookupSource>(name, lookupActorFactory);
55+
factory.RegisterSource<Generic::TSource>(name, readActorFactory);
56+
factory.RegisterLookupSource<Generic::TLookupSource>(name, lookupActorFactory);
5757
}
5858
}
5959

‎ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,10 @@ namespace NYql::NDq {
4545
TCollectStatsLevel statsLevel,
4646
NConnector::IClient::TPtr client,
4747
TGenericTokenProvider::TPtr tokenProvider,
48-
NGeneric::TSource&& source,
48+
Generic::TSource&& source,
4949
const NActors::TActorId& computeActorId,
5050
const NKikimr::NMiniKQL::THolderFactory& holderFactory,
51-
TVector<NGeneric::TPartition>&& partitions)
51+
TVector<Generic::TPartition>&& partitions)
5252
: InputIndex_(inputIndex)
5353
, ComputeActorId_(computeActorId)
5454
, Client_(std::move(client))
@@ -380,32 +380,32 @@ namespace NYql::NDq {
380380
NConnector::IClient::TPtr Client_;
381381
TGenericTokenProvider::TPtr TokenProvider_;
382382

383-
TVector<NGeneric::TPartition> Partitions_;
383+
TVector<Generic::TPartition> Partitions_;
384384

385385
NConnector::IReadSplitsStreamIterator::TPtr ReadSplitsIterator_;
386386
std::optional<NConnector::NApi::TReadSplitsResponse> LastReadSplitsResponse_;
387387
bool ReadSplitsFinished_ = false;
388388

389389
NKikimr::NMiniKQL::TPlainContainerCache ArrowRowContainerCache_;
390390
const NKikimr::NMiniKQL::THolderFactory& HolderFactory_;
391-
NGeneric::TSource Source_;
391+
Generic::TSource Source_;
392392
};
393393

394394
void ExtractPartitionsFromParams(
395-
TVector<NGeneric::TPartition>& partitions,
395+
TVector<Generic::TPartition>& partitions,
396396
const THashMap<TString, TString>& taskParams, // partitions are here in v1
397397
const TVector<TString>& readRanges // partitions are here in v2
398398
) {
399399
if (!readRanges.empty()) {
400400
for (const auto& readRange : readRanges) {
401-
NGeneric::TPartition partition;
401+
Generic::TPartition partition;
402402
YQL_ENSURE(partition.ParseFromString(readRange), "Failed to parse partition from read ranges");
403403
partitions.emplace_back(std::move(partition));
404404
}
405405
} else {
406406
const auto& iter = taskParams.find(GenericProviderName);
407407
if (iter != taskParams.end()) {
408-
NGeneric::TPartition partition;
408+
Generic::TPartition partition;
409409
TStringInput input(iter->first);
410410
YQL_ENSURE(partition.ParseFromString(iter->second), "Failed to parse partition from task params");
411411
partitions.emplace_back(std::move(partition));
@@ -417,7 +417,7 @@ namespace NYql::NDq {
417417

418418
std::pair<NYql::NDq::IDqComputeActorAsyncInput*, IActor*>
419419
CreateGenericReadActor(NConnector::IClient::TPtr genericClient,
420-
NGeneric::TSource&& source,
420+
Generic::TSource&& source,
421421
ui64 inputIndex,
422422
TCollectStatsLevel statsLevel,
423423
const THashMap<TString, TString>& /*secureParams*/,
@@ -428,7 +428,7 @@ namespace NYql::NDq {
428428
ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
429429
const NKikimr::NMiniKQL::THolderFactory& holderFactory)
430430
{
431-
TVector<NGeneric::TPartition> partitions;
431+
TVector<Generic::TPartition> partitions;
432432
ExtractPartitionsFromParams(partitions, taskParams, readRanges);
433433

434434
const auto dsi = source.select().data_source_instance();

‎ydb/library/yql/providers/generic/actors/yql_generic_read_actor.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ namespace NYql::NDq {
1111
std::pair<NYql::NDq::IDqComputeActorAsyncInput*, NActors::IActor*>
1212
CreateGenericReadActor(
1313
NConnector::IClient::TPtr genericClient,
14-
NGeneric::TSource&& params,
14+
Generic::TSource&& params,
1515
ui64 inputIndex,
1616
TCollectStatsLevel statsLevel,
1717
const THashMap<TString, TString>& secureParams,

‎ydb/library/yql/providers/generic/proto/partition.proto

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ syntax = "proto3";
22

33
option cc_enable_arenas = true;
44

5-
package NYql.NGeneric;
5+
// NOTE: never try to rename this package as it will break compatibility in DQ.
6+
package NYql.Generic;
67

78
import "ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto";
89

‎ydb/library/yql/providers/generic/proto/source.proto

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ syntax = "proto3";
22

33
option cc_enable_arenas = true;
44

5-
package NYql.NGeneric;
5+
// NOTE: never try to rename this package as it will break compatibility in DQ.
6+
package NYql.Generic;
67

78
import "ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto";
89
import "yql/essentials/providers/common/proto/gateways_config.proto";

‎ydb/library/yql/providers/generic/provider/ut/pushdown/pushdown_ut.cpp

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ struct TFakeGenericClient: public NConnector::IClient {
169169

170170
class TBuildDqSourceSettingsTransformer: public TOptimizeTransformerBase {
171171
public:
172-
explicit TBuildDqSourceSettingsTransformer(TTypeAnnotationContext* types, NGeneric::TSource* dqSourceSettings, bool* dqSourceSettingsWereBuilt)
172+
explicit TBuildDqSourceSettingsTransformer(TTypeAnnotationContext* types, Generic::TSource* dqSourceSettings, bool* dqSourceSettingsWereBuilt)
173173
: TOptimizeTransformerBase(types, NLog::EComponent::ProviderGeneric, {})
174174
, DqSourceSettings_(dqSourceSettings)
175175
, DqSourceSettingsWereBuilt_(dqSourceSettingsWereBuilt)
@@ -207,13 +207,13 @@ class TBuildDqSourceSettingsTransformer: public TOptimizeTransformerBase {
207207
TString sourceType;
208208
dqIntegration->FillSourceSettings(*dqSourceNode, settings, sourceType, 1, ctx);
209209
UNIT_ASSERT_STRINGS_EQUAL(sourceType, "PostgreSqlGeneric");
210-
UNIT_ASSERT(settings.Is<NGeneric::TSource>());
210+
UNIT_ASSERT(settings.Is<Generic::TSource>());
211211
settings.UnpackTo(DqSourceSettings_);
212212
*DqSourceSettingsWereBuilt_ = true;
213213
}
214214

215215
private:
216-
NGeneric::TSource* DqSourceSettings_;
216+
Generic::TSource* DqSourceSettings_;
217217
bool* DqSourceSettingsWereBuilt_;
218218
};
219219

@@ -232,7 +232,7 @@ struct TPushdownFixture: public NUnitTest::TBaseFixture {
232232

233233
TAutoPtr<IGraphTransformer> Transformer;
234234
TAutoPtr<IGraphTransformer> BuildDqSourceSettingsTransformer;
235-
NGeneric::TSource DqSourceSettings;
235+
Generic::TSource DqSourceSettings;
236236
bool DqSourceSettingsWereBuilt = false;
237237

238238
TExprNode::TPtr InitialExprRoot;
@@ -352,7 +352,7 @@ struct TPushdownFixture: public NUnitTest::TBaseFixture {
352352
void AssertFilter(const TString& lambdaText, const TString& filterText) {
353353
const auto& filter = BuildProtoFilterFromLambda(lambdaText);
354354
NConnector::NApi::TPredicate expectedFilter;
355-
UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(filterText, &expectedFilter));
355+
UNIT_ASSERT_C(google::protobuf::TextFormat::ParseFromString(filterText, &expectedFilter), expectedFilter.InitializationErrorString());
356356
UNIT_ASSERT_STRINGS_EQUAL(filter.Utf8DebugString(), expectedFilter.Utf8DebugString());
357357
}
358358

@@ -657,7 +657,7 @@ Y_UNIT_TEST_SUITE_F(PushdownTest, TPushdownFixture) {
657657
}
658658

659659
Y_UNIT_TEST(StringFieldsNotSupported) {
660-
AssertNoPush(
660+
AssertFilter(
661661
// Note that R"ast()ast" is empty string!
662662
R"ast(
663663
(Coalesce
@@ -667,17 +667,48 @@ Y_UNIT_TEST_SUITE_F(PushdownTest, TPushdownFixture) {
667667
)
668668
(Bool '"false")
669669
)
670-
)ast");
670+
)ast",
671+
R"proto(
672+
comparison {
673+
operation: EQ
674+
left_value {
675+
column: "col_utf8"
676+
}
677+
right_value {
678+
column: "col_optional_utf8"
679+
}
680+
}
681+
)proto"
682+
);
671683
}
672684

673685
Y_UNIT_TEST(StringFieldsNotSupported2) {
674-
AssertNoPush(
686+
AssertFilter(
675687
// Note that R"ast()ast" is empty string!
676688
R"ast(
677689
(!=
678690
(Member $row '"col_string")
679691
(String '"value")
680692
)
681-
)ast");
693+
)ast",
694+
R"proto(
695+
comparison {
696+
operation: NE
697+
left_value {
698+
column: "col_string"
699+
}
700+
right_value {
701+
typed_value {
702+
type {
703+
type_id: STRING
704+
}
705+
value {
706+
bytes_value: "value"
707+
}
708+
}
709+
}
710+
}
711+
)proto"
712+
);
682713
}
683714
}

‎ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ namespace NYql {
142142
if (totalSplits <= partitionSettings.MaxPartitions) {
143143
// If there are not too many splits, simply make a single-split partitions.
144144
for (size_t i = 0; i < totalSplits; i++) {
145-
NGeneric::TPartition partition;
145+
Generic::TPartition partition;
146146
*partition.add_splits() = tableMeta->Splits[i];
147147
TString partitionStr;
148148
YQL_ENSURE(partition.SerializeToString(&partitionStr), "Failed to serialize partition");
@@ -154,7 +154,7 @@ namespace NYql {
154154
size_t splitsPerPartition = (totalSplits / partitionSettings.MaxPartitions - 1) + 1;
155155

156156
for (size_t i = 0; i < totalSplits; i += splitsPerPartition) {
157-
NGeneric::TPartition partition;
157+
Generic::TPartition partition;
158158
for (size_t j = i; j < i + splitsPerPartition && j < totalSplits; j++) {
159159
*partition.add_splits() = tableMeta->Splits[j];
160160
}
@@ -178,7 +178,7 @@ namespace NYql {
178178
const auto& clusterConfig = State_->Configuration->ClusterNamesToClusterConfigs[clusterName];
179179
const auto& endpoint = clusterConfig.endpoint();
180180

181-
NGeneric::TSource source;
181+
Generic::TSource source;
182182

183183
YQL_CLOG(INFO, ProviderGeneric)
184184
<< "Filling source settings"
@@ -337,7 +337,7 @@ namespace NYql {
337337
throw yexception() << "Get table metadata: " << issues.ToOneLineString();
338338
}
339339

340-
NGeneric::TLookupSource source;
340+
Generic::TLookupSource source;
341341
source.set_table(tableName);
342342
*source.mutable_data_source_instance() = tableMeta->DataSourceInstance;
343343

0 commit comments

Comments
 (0)