Skip to content

Commit 514a0c4

Browse files
vitalyisaev2vitalyisaev2
authored
Merge from main (ydb-platform#12647)
1 parent e98b6b8 commit 514a0c4

File tree

107 files changed

+716
-727
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

107 files changed

+716
-727
lines changed

‎ydb/core/fq/libs/actors/clusters_from_connections.cpp

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#include "clusters_from_connections.h"
22

33
#include <yql/essentials/providers/common/provider/yql_provider_names.h>
4-
#include <ydb/library/yql/providers/generic/connector/api/common/data_source.pb.h>
4+
#include <yql/essentials/providers/common/proto/gateways_config.pb.h>
55
#include <ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.h>
66
#include <yql/essentials/utils/url_builder.h>
77
#include <ydb/library/actors/http/http.h>
@@ -112,7 +112,7 @@ void FillGenericClusterConfigBase(
112112
TGenericClusterConfig& clusterCfg,
113113
const TConnection& connection,
114114
const TString& connectionName,
115-
NConnector::NApi::EDataSourceKind dataSourceKind,
115+
NYql::EGenericDataSourceKind dataSourceKind,
116116
const TString& authToken,
117117
const THashMap<TString, TString>& accountIdSignatures
118118
) {
@@ -128,21 +128,21 @@ void FillGenericClusterConfigBase(
128128
// In YQv1 we just hardcode the appropriate protocols here.
129129
// In YQv2 protocol can be configured via `CREATE EXTERNAL DATA SOURCE` params.
130130
switch (dataSourceKind) {
131-
case NYql::NConnector::NApi::CLICKHOUSE:
132-
clusterCfg.SetProtocol(common.GetUseNativeProtocolForClickHouse() ? NYql::NConnector::NApi::EProtocol::NATIVE : NYql::NConnector::NApi::EProtocol::HTTP);
131+
case NYql::EGenericDataSourceKind::CLICKHOUSE:
132+
clusterCfg.SetProtocol(common.GetUseNativeProtocolForClickHouse() ? NYql::EGenericProtocol::NATIVE : NYql::EGenericProtocol::HTTP);
133133
break;
134-
case NYql::NConnector::NApi::GREENPLUM:
135-
clusterCfg.SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE);
134+
case NYql::EGenericDataSourceKind::GREENPLUM:
135+
clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE);
136136
break;
137-
case NYql::NConnector::NApi::MYSQL:
138-
clusterCfg.SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE);
137+
case NYql::EGenericDataSourceKind::MYSQL:
138+
clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE);
139139
break;
140-
case NYql::NConnector::NApi::POSTGRESQL:
141-
clusterCfg.SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE);
140+
case NYql::EGenericDataSourceKind::POSTGRESQL:
141+
clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE);
142142
break;
143143
default:
144144
ythrow yexception() << "Unexpected data source kind: '"
145-
<< NYql::NConnector::NApi::EDataSourceKind_Name(dataSourceKind) << "'";
145+
<< NYql::EGenericDataSourceKind_Name(dataSourceKind) << "'";
146146
}
147147

148148
ValidateGenericClusterConfig(clusterCfg, "NFq::FillGenericClusterFromConfig");
@@ -154,7 +154,7 @@ void FillGenericClusterConfig(
154154
TGenericClusterConfig& clusterCfg,
155155
const TConnection& connection,
156156
const TString& connectionName,
157-
NConnector::NApi::EDataSourceKind dataSourceKind,
157+
NYql::EGenericDataSourceKind dataSourceKind,
158158
const TString& authToken,
159159
const THashMap<TString, TString>& accountIdSignatures
160160
) {
@@ -167,7 +167,7 @@ void FillGenericClusterConfig<FederatedQuery::PostgreSQLCluster>(
167167
TGenericClusterConfig& clusterCfg,
168168
const FederatedQuery::PostgreSQLCluster& connection,
169169
const TString& connectionName,
170-
NConnector::NApi::EDataSourceKind dataSourceKind,
170+
NYql::EGenericDataSourceKind dataSourceKind,
171171
const TString& authToken,
172172
const THashMap<TString, TString>& accountIdSignatures
173173
){
@@ -227,8 +227,8 @@ void AddClustersFromConnections(
227227
case FederatedQuery::ConnectionSetting::kYdbDatabase: {
228228
const auto& db = conn.content().setting().ydb_database();
229229
auto* clusterCfg = gatewaysConfig.MutableGeneric()->AddClusterMapping();
230-
clusterCfg->SetKind(NYql::NConnector::NApi::EDataSourceKind::YDB);
231-
clusterCfg->SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE);
230+
clusterCfg->SetKind(NYql::EGenericDataSourceKind::YDB);
231+
clusterCfg->SetProtocol(NYql::EGenericProtocol::NATIVE);
232232
clusterCfg->SetName(connectionName);
233233
clusterCfg->SetDatabaseId(db.database_id());
234234
clusterCfg->SetUseSsl(!common.GetDisableSslForGenericDataSources());
@@ -242,7 +242,7 @@ void AddClustersFromConnections(
242242
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
243243
conn.content().setting().clickhouse_cluster(),
244244
connectionName,
245-
NYql::NConnector::NApi::EDataSourceKind::CLICKHOUSE,
245+
NYql::EGenericDataSourceKind::CLICKHOUSE,
246246
authToken,
247247
accountIdSignatures);
248248
clusters.emplace(connectionName, GenericProviderName);
@@ -275,7 +275,7 @@ void AddClustersFromConnections(
275275
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
276276
conn.content().setting().postgresql_cluster(),
277277
connectionName,
278-
NYql::NConnector::NApi::EDataSourceKind::POSTGRESQL,
278+
NYql::EGenericDataSourceKind::POSTGRESQL,
279279
authToken,
280280
accountIdSignatures);
281281
clusters.emplace(connectionName, GenericProviderName);
@@ -287,7 +287,7 @@ void AddClustersFromConnections(
287287
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
288288
conn.content().setting().greenplum_cluster(),
289289
connectionName,
290-
NYql::NConnector::NApi::EDataSourceKind::GREENPLUM,
290+
NYql::EGenericDataSourceKind::GREENPLUM,
291291
authToken,
292292
accountIdSignatures);
293293
clusters.emplace(connectionName, GenericProviderName);
@@ -299,7 +299,7 @@ void AddClustersFromConnections(
299299
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
300300
conn.content().setting().mysql_cluster(),
301301
connectionName,
302-
NYql::NConnector::NApi::EDataSourceKind::MYSQL,
302+
NYql::EGenericDataSourceKind::MYSQL,
303303
authToken,
304304
accountIdSignatures);
305305
clusters.emplace(connectionName, GenericProviderName);
@@ -308,7 +308,7 @@ void AddClustersFromConnections(
308308
case FederatedQuery::ConnectionSetting::kLogging: {
309309
const auto& connection = conn.content().setting().logging();
310310
auto* clusterCfg = gatewaysConfig.MutableGeneric()->AddClusterMapping();
311-
clusterCfg->SetKind(NYql::NConnector::NApi::EDataSourceKind::LOGGING);
311+
clusterCfg->SetKind(NYql::EGenericDataSourceKind::LOGGING);
312312
clusterCfg->SetName(connectionName);
313313
clusterCfg->mutable_datasourceoptions()->insert({"folder_id", connection.folder_id()});
314314
FillClusterAuth(*clusterCfg, connection.auth(), authToken, accountIdSignatures);

‎ydb/core/fq/libs/actors/database_resolver.cpp

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ using TParser = std::function<TDatabaseDescription(
2929
NJson::TJsonValue& body,
3030
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
3131
bool useTls,
32-
NConnector::NApi::EProtocol protocol
32+
NYql::EGenericProtocol protocol
3333
)>;
3434
using TParsers = THashMap<NYql::EDatabaseType, TParser>;
3535

@@ -292,7 +292,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
292292
.SetErrorTtl(TDuration::Minutes(1))
293293
.SetMaxSize(1000000))
294294
{
295-
auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool, NConnector::NApi::EProtocol) {
295+
auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool, NYql::EGenericProtocol) {
296296
bool secure = false;
297297
TString endpoint = databaseInfo.GetMap().at("endpoint").GetStringRobust();
298298
TString prefix("/?database=");
@@ -333,7 +333,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
333333
NJson::TJsonValue& databaseInfo,
334334
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
335335
bool useTls,
336-
NConnector::NApi::EProtocol protocol)
336+
NYql::EGenericProtocol protocol)
337337
{
338338
auto ret = ydbParser(databaseInfo, mdbEndpointGenerator, useTls, protocol);
339339
// TODO: Take explicit field from MVP
@@ -349,7 +349,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
349349
NJson::TJsonValue& databaseInfo,
350350
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
351351
bool useTls,
352-
NConnector::NApi::EProtocol protocol
352+
NYql::EGenericProtocol protocol
353353
) {
354354
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
355355
TVector<TString> aliveHosts;
@@ -380,7 +380,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
380380
NJson::TJsonValue& databaseInfo,
381381
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
382382
bool useTls,
383-
NConnector::NApi::EProtocol protocol
383+
NYql::EGenericProtocol protocol
384384
) {
385385
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
386386
TVector<TString> aliveHosts;
@@ -427,7 +427,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
427427
NJson::TJsonValue& databaseInfo,
428428
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
429429
bool useTls,
430-
NConnector::NApi::EProtocol protocol
430+
NYql::EGenericProtocol protocol
431431
) {
432432
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
433433
TString aliveHost;
@@ -465,7 +465,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
465465
NJson::TJsonValue& databaseInfo,
466466
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
467467
bool useTls,
468-
NConnector::NApi::EProtocol protocol
468+
NYql::EGenericProtocol protocol
469469
) {
470470
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
471471
TVector<TString> aliveHosts;

‎ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
124124

125125
void Test(
126126
NYql::EDatabaseType databaseType,
127-
NYql::NConnector::NApi::EProtocol protocol,
127+
NYql::EGenericProtocol protocol,
128128
const TString& getUrl,
129129
const TString& status,
130130
const TString& responseBody,
@@ -186,7 +186,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
186186
Y_UNIT_TEST(Ydb_Serverless) {
187187
Test(
188188
NYql::EDatabaseType::Ydb,
189-
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
189+
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
190190
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
191191
"200",
192192
R"(
@@ -218,7 +218,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
218218

219219
Test(
220220
NYql::EDatabaseType::Ydb,
221-
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
221+
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
222222
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
223223
"",
224224
"",
@@ -237,7 +237,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
237237
Y_UNIT_TEST(Ydb_Dedicated) {
238238
Test(
239239
NYql::EDatabaseType::Ydb,
240-
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
240+
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
241241
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
242242
"200",
243243
R"(
@@ -259,7 +259,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
259259
Y_UNIT_TEST(DataStreams_Serverless) {
260260
Test(
261261
NYql::EDatabaseType::DataStreams,
262-
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
262+
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
263263
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
264264
"200",
265265
R"(
@@ -280,7 +280,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
280280
Y_UNIT_TEST(DataStreams_Dedicated) {
281281
Test(
282282
NYql::EDatabaseType::DataStreams,
283-
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
283+
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
284284
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
285285
"200",
286286
R"(
@@ -302,7 +302,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
302302
Y_UNIT_TEST(ClickHouseNative) {
303303
Test(
304304
NYql::EDatabaseType::ClickHouse,
305-
NYql::NConnector::NApi::EProtocol::NATIVE,
305+
NYql::EGenericProtocol::NATIVE,
306306
"https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts",
307307
"200",
308308
R"({
@@ -336,7 +336,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
336336
Y_UNIT_TEST(ClickHouseHttp) {
337337
Test(
338338
NYql::EDatabaseType::ClickHouse,
339-
NYql::NConnector::NApi::EProtocol::HTTP,
339+
NYql::EGenericProtocol::HTTP,
340340
"https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts",
341341
"200",
342342
R"({
@@ -381,7 +381,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
381381

382382
Test(
383383
NYql::EDatabaseType::ClickHouse,
384-
NYql::NConnector::NApi::EProtocol::HTTP,
384+
NYql::EGenericProtocol::HTTP,
385385
"https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts",
386386
"403",
387387
R"(
@@ -405,7 +405,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
405405
Y_UNIT_TEST(PostgreSQL) {
406406
Test(
407407
NYql::EDatabaseType::PostgreSQL,
408-
NYql::NConnector::NApi::EProtocol::NATIVE,
408+
NYql::EGenericProtocol::NATIVE,
409409
"https://mdb.api.cloud.yandex.net:443/managed-postgresql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
410410
"200",
411411
R"({
@@ -454,7 +454,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
454454

455455
Test(
456456
NYql::EDatabaseType::PostgreSQL,
457-
NYql::NConnector::NApi::EProtocol::NATIVE,
457+
NYql::EGenericProtocol::NATIVE,
458458
"https://mdb.api.cloud.yandex.net:443/managed-postgresql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
459459
"403",
460460
R"(
@@ -478,7 +478,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
478478
Y_UNIT_TEST(Greenplum_MasterNode) {
479479
Test(
480480
NYql::EDatabaseType::Greenplum,
481-
NYql::NConnector::NApi::EProtocol::NATIVE,
481+
NYql::EGenericProtocol::NATIVE,
482482
"https://mdb.api.cloud.yandex.net:443/managed-greenplum/v1/clusters/etn021us5r9rhld1vgbh/master-hosts",
483483
"200",
484484
R"({
@@ -520,7 +520,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
520520

521521
Test(
522522
NYql::EDatabaseType::Greenplum,
523-
NYql::NConnector::NApi::EProtocol::NATIVE,
523+
NYql::EGenericProtocol::NATIVE,
524524
"https://mdb.api.cloud.yandex.net:443/managed-greenplum/v1/clusters/etn021us5r9rhld1vgbh/master-hosts",
525525
"403",
526526
R"(
@@ -542,7 +542,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
542542
Y_UNIT_TEST(MySQL) {
543543
Test(
544544
NYql::EDatabaseType::MySQL,
545-
NYql::NConnector::NApi::EProtocol::NATIVE,
545+
NYql::EGenericProtocol::NATIVE,
546546
"https://mdb.api.cloud.yandex.net:443/managed-mysql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
547547
"200",
548548
R"({
@@ -590,7 +590,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
590590

591591
Test(
592592
NYql::EDatabaseType::MySQL,
593-
NYql::NConnector::NApi::EProtocol::NATIVE,
593+
NYql::EGenericProtocol::NATIVE,
594594
"https://mdb.api.cloud.yandex.net:443/managed-mysql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
595595
"403",
596596
R"(
@@ -624,7 +624,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
624624
};
625625
Test(
626626
NYql::EDatabaseType::DataStreams,
627-
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
627+
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
628628
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
629629
"403",
630630
R"(
@@ -642,7 +642,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
642642

643643
NYql::TDatabaseAuth databaseAuth;
644644
databaseAuth.UseTls = true;
645-
databaseAuth.Protocol = NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED;
645+
databaseAuth.Protocol = NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED;
646646

647647
TString databaseId1{"etn021us5r9rhld1vgb1"};
648648
TString databaseId2{"etn021us5r9rhld1vgb2"};

‎ydb/core/fq/libs/actors/ya.make

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@ PEERDIR(
7272
ydb/library/yql/providers/dq/provider
7373
ydb/library/yql/providers/dq/provider/exec
7474
ydb/library/yql/providers/dq/worker_manager/interface
75-
ydb/library/yql/providers/generic/connector/api/common
7675
ydb/library/yql/providers/generic/connector/libcpp
7776
ydb/library/yql/providers/generic/provider
7877
ydb/library/yql/providers/pq/cm_client

0 commit comments

Comments
 (0)