Skip to content

Commit 6097a78

Browse files
vitalyisaev2vitalyisaev2
authored
Merge from main (#12612)
1 parent 6bb4b1c commit 6097a78

File tree

79 files changed

+627
-427
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

79 files changed

+627
-427
lines changed

‎ydb/core/fq/libs/actors/clusters_from_connections.cpp

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#include "clusters_from_connections.h"
22

33
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
4-
#include <ydb/library/yql/providers/generic/connector/api/common/data_source.pb.h>
4+
#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
55
#include <ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.h>
66
#include <ydb/library/yql/utils/url_builder.h>
77
#include <ydb/library/actors/http/http.h>
@@ -109,7 +109,7 @@ void FillGenericClusterConfigBase(
109109
TGenericClusterConfig& clusterCfg,
110110
const TConnection& connection,
111111
const TString& connectionName,
112-
NConnector::NApi::EDataSourceKind dataSourceKind,
112+
NYql::EGenericDataSourceKind dataSourceKind,
113113
const TString& authToken,
114114
const THashMap<TString, TString>& accountIdSignatures
115115
) {
@@ -125,15 +125,21 @@ void FillGenericClusterConfigBase(
125125
// In YQv1 we just hardcode desired protocols here.
126126
// In YQv2 protocol can be configured via `CREATE EXTERNAL DATA SOURCE` params.
127127
switch (dataSourceKind) {
128-
case NYql::NConnector::NApi::CLICKHOUSE:
129-
clusterCfg.SetProtocol(common.GetUseNativeProtocolForClickHouse() ? NYql::NConnector::NApi::EProtocol::NATIVE : NYql::NConnector::NApi::EProtocol::HTTP);
128+
case NYql::EGenericDataSourceKind::CLICKHOUSE:
129+
clusterCfg.SetProtocol(common.GetUseNativeProtocolForClickHouse() ? NYql::EGenericProtocol::NATIVE : NYql::EGenericProtocol::HTTP);
130130
break;
131-
case NYql::NConnector::NApi::POSTGRESQL:
132-
clusterCfg.SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE);
131+
case NYql::EGenericDataSourceKind::GREENPLUM:
132+
clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE);
133+
break;
134+
case NYql::EGenericDataSourceKind::MYSQL:
135+
clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE);
136+
break;
137+
case NYql::EGenericDataSourceKind::POSTGRESQL:
138+
clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE);
133139
break;
134140
default:
135141
ythrow yexception() << "Unexpected data source kind: '"
136-
<< NYql::NConnector::NApi::EDataSourceKind_Name(dataSourceKind) << "'";
142+
<< NYql::EGenericDataSourceKind_Name(dataSourceKind) << "'";
137143
}
138144

139145
ValidateGenericClusterConfig(clusterCfg, "NFq::FillGenericClusterFromConfig");
@@ -145,7 +151,7 @@ void FillGenericClusterConfig(
145151
TGenericClusterConfig& clusterCfg,
146152
const TConnection& connection,
147153
const TString& connectionName,
148-
NConnector::NApi::EDataSourceKind dataSourceKind,
154+
NYql::EGenericDataSourceKind dataSourceKind,
149155
const TString& authToken,
150156
const THashMap<TString, TString>& accountIdSignatures
151157
) {
@@ -158,7 +164,7 @@ void FillGenericClusterConfig<FederatedQuery::PostgreSQLCluster>(
158164
TGenericClusterConfig& clusterCfg,
159165
const FederatedQuery::PostgreSQLCluster& connection,
160166
const TString& connectionName,
161-
NConnector::NApi::EDataSourceKind dataSourceKind,
167+
NYql::EGenericDataSourceKind dataSourceKind,
162168
const TString& authToken,
163169
const THashMap<TString, TString>& accountIdSignatures
164170
){
@@ -217,8 +223,8 @@ void AddClustersFromConnections(
217223
case FederatedQuery::ConnectionSetting::kYdbDatabase: {
218224
const auto& db = conn.content().setting().ydb_database();
219225
auto* clusterCfg = gatewaysConfig.MutableGeneric()->AddClusterMapping();
220-
clusterCfg->SetKind(NYql::NConnector::NApi::EDataSourceKind::YDB);
221-
clusterCfg->SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE);
226+
clusterCfg->SetKind(NYql::EGenericDataSourceKind::YDB);
227+
clusterCfg->SetProtocol(NYql::EGenericProtocol::NATIVE);
222228
clusterCfg->SetName(connectionName);
223229
clusterCfg->SetDatabaseId(db.database_id());
224230
clusterCfg->SetUseSsl(!common.GetDisableSslForGenericDataSources());
@@ -232,7 +238,7 @@ void AddClustersFromConnections(
232238
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
233239
conn.content().setting().clickhouse_cluster(),
234240
connectionName,
235-
NYql::NConnector::NApi::EDataSourceKind::CLICKHOUSE,
241+
NYql::EGenericDataSourceKind::CLICKHOUSE,
236242
authToken,
237243
accountIdSignatures);
238244
clusters.emplace(connectionName, GenericProviderName);
@@ -265,7 +271,7 @@ void AddClustersFromConnections(
265271
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
266272
conn.content().setting().postgresql_cluster(),
267273
connectionName,
268-
NYql::NConnector::NApi::EDataSourceKind::POSTGRESQL,
274+
NYql::EGenericDataSourceKind::POSTGRESQL,
269275
authToken,
270276
accountIdSignatures);
271277
clusters.emplace(connectionName, GenericProviderName);
@@ -277,7 +283,7 @@ void AddClustersFromConnections(
277283
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
278284
conn.content().setting().greenplum_cluster(),
279285
connectionName,
280-
NYql::NConnector::NApi::EDataSourceKind::GREENPLUM,
286+
NYql::EGenericDataSourceKind::GREENPLUM,
281287
authToken,
282288
accountIdSignatures);
283289
clusters.emplace(connectionName, GenericProviderName);
@@ -289,7 +295,7 @@ void AddClustersFromConnections(
289295
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
290296
conn.content().setting().mysql_cluster(),
291297
connectionName,
292-
NYql::NConnector::NApi::EDataSourceKind::MYSQL,
298+
NYql::EGenericDataSourceKind::MYSQL,
293299
authToken,
294300
accountIdSignatures);
295301
clusters.emplace(connectionName, GenericProviderName);
@@ -298,7 +304,7 @@ void AddClustersFromConnections(
298304
case FederatedQuery::ConnectionSetting::kLogging: {
299305
const auto& connection = conn.content().setting().logging();
300306
auto* clusterCfg = gatewaysConfig.MutableGeneric()->AddClusterMapping();
301-
clusterCfg->SetKind(NYql::NConnector::NApi::EDataSourceKind::LOGGING);
307+
clusterCfg->SetKind(NYql::EGenericDataSourceKind::LOGGING);
302308
clusterCfg->SetName(connectionName);
303309
clusterCfg->mutable_datasourceoptions()->insert({"folder_id", connection.folder_id()});
304310
FillClusterAuth(*clusterCfg, connection.auth(), authToken, accountIdSignatures);

‎ydb/core/fq/libs/actors/database_resolver.cpp

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ using TParser = std::function<TDatabaseDescription(
2929
NJson::TJsonValue& body,
3030
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
3131
bool useTls,
32-
NConnector::NApi::EProtocol protocol
32+
NYql::EGenericProtocol protocol
3333
)>;
3434
using TParsers = THashMap<NYql::EDatabaseType, TParser>;
3535

@@ -292,7 +292,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
292292
.SetErrorTtl(TDuration::Minutes(1))
293293
.SetMaxSize(1000000))
294294
{
295-
auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool, NConnector::NApi::EProtocol) {
295+
auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool, NYql::EGenericProtocol) {
296296
bool secure = false;
297297
TString endpoint = databaseInfo.GetMap().at("endpoint").GetStringRobust();
298298
TString prefix("/?database=");
@@ -333,7 +333,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
333333
NJson::TJsonValue& databaseInfo,
334334
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
335335
bool useTls,
336-
NConnector::NApi::EProtocol protocol)
336+
NYql::EGenericProtocol protocol)
337337
{
338338
auto ret = ydbParser(databaseInfo, mdbEndpointGenerator, useTls, protocol);
339339
// TODO: Take explicit field from MVP
@@ -349,7 +349,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
349349
NJson::TJsonValue& databaseInfo,
350350
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
351351
bool useTls,
352-
NConnector::NApi::EProtocol protocol
352+
NYql::EGenericProtocol protocol
353353
) {
354354
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
355355
TVector<TString> aliveHosts;
@@ -380,7 +380,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
380380
NJson::TJsonValue& databaseInfo,
381381
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
382382
bool useTls,
383-
NConnector::NApi::EProtocol protocol
383+
NYql::EGenericProtocol protocol
384384
) {
385385
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
386386
TVector<TString> aliveHosts;
@@ -427,7 +427,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
427427
NJson::TJsonValue& databaseInfo,
428428
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
429429
bool useTls,
430-
NConnector::NApi::EProtocol protocol
430+
NYql::EGenericProtocol protocol
431431
) {
432432
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
433433
TString aliveHost;
@@ -465,7 +465,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
465465
NJson::TJsonValue& databaseInfo,
466466
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
467467
bool useTls,
468-
NConnector::NApi::EProtocol protocol
468+
NYql::EGenericProtocol protocol
469469
) {
470470
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
471471
TVector<TString> aliveHosts;

‎ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
124124

125125
void Test(
126126
NYql::EDatabaseType databaseType,
127-
NYql::NConnector::NApi::EProtocol protocol,
127+
NYql::EGenericProtocol protocol,
128128
const TString& getUrl,
129129
const TString& status,
130130
const TString& responseBody,
@@ -186,7 +186,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
186186
Y_UNIT_TEST(Ydb_Serverless) {
187187
Test(
188188
NYql::EDatabaseType::Ydb,
189-
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
189+
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
190190
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
191191
"200",
192192
R"(
@@ -218,7 +218,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
218218

219219
Test(
220220
NYql::EDatabaseType::Ydb,
221-
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
221+
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
222222
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
223223
"",
224224
"",
@@ -237,7 +237,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
237237
Y_UNIT_TEST(Ydb_Dedicated) {
238238
Test(
239239
NYql::EDatabaseType::Ydb,
240-
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
240+
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
241241
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
242242
"200",
243243
R"(
@@ -259,7 +259,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
259259
Y_UNIT_TEST(DataStreams_Serverless) {
260260
Test(
261261
NYql::EDatabaseType::DataStreams,
262-
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
262+
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
263263
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
264264
"200",
265265
R"(
@@ -280,7 +280,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
280280
Y_UNIT_TEST(DataStreams_Dedicated) {
281281
Test(
282282
NYql::EDatabaseType::DataStreams,
283-
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
283+
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
284284
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
285285
"200",
286286
R"(
@@ -302,7 +302,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
302302
Y_UNIT_TEST(ClickHouseNative) {
303303
Test(
304304
NYql::EDatabaseType::ClickHouse,
305-
NYql::NConnector::NApi::EProtocol::NATIVE,
305+
NYql::EGenericProtocol::NATIVE,
306306
"https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts",
307307
"200",
308308
R"({
@@ -336,7 +336,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
336336
Y_UNIT_TEST(ClickHouseHttp) {
337337
Test(
338338
NYql::EDatabaseType::ClickHouse,
339-
NYql::NConnector::NApi::EProtocol::HTTP,
339+
NYql::EGenericProtocol::HTTP,
340340
"https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts",
341341
"200",
342342
R"({
@@ -381,7 +381,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
381381

382382
Test(
383383
NYql::EDatabaseType::ClickHouse,
384-
NYql::NConnector::NApi::EProtocol::HTTP,
384+
NYql::EGenericProtocol::HTTP,
385385
"https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts",
386386
"403",
387387
R"(
@@ -405,7 +405,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
405405
Y_UNIT_TEST(PostgreSQL) {
406406
Test(
407407
NYql::EDatabaseType::PostgreSQL,
408-
NYql::NConnector::NApi::EProtocol::NATIVE,
408+
NYql::EGenericProtocol::NATIVE,
409409
"https://mdb.api.cloud.yandex.net:443/managed-postgresql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
410410
"200",
411411
R"({
@@ -454,7 +454,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
454454

455455
Test(
456456
NYql::EDatabaseType::PostgreSQL,
457-
NYql::NConnector::NApi::EProtocol::NATIVE,
457+
NYql::EGenericProtocol::NATIVE,
458458
"https://mdb.api.cloud.yandex.net:443/managed-postgresql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
459459
"403",
460460
R"(
@@ -478,7 +478,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
478478
Y_UNIT_TEST(Greenplum_MasterNode) {
479479
Test(
480480
NYql::EDatabaseType::Greenplum,
481-
NYql::NConnector::NApi::EProtocol::NATIVE,
481+
NYql::EGenericProtocol::NATIVE,
482482
"https://mdb.api.cloud.yandex.net:443/managed-greenplum/v1/clusters/etn021us5r9rhld1vgbh/master-hosts",
483483
"200",
484484
R"({
@@ -520,7 +520,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
520520

521521
Test(
522522
NYql::EDatabaseType::Greenplum,
523-
NYql::NConnector::NApi::EProtocol::NATIVE,
523+
NYql::EGenericProtocol::NATIVE,
524524
"https://mdb.api.cloud.yandex.net:443/managed-greenplum/v1/clusters/etn021us5r9rhld1vgbh/master-hosts",
525525
"403",
526526
R"(
@@ -542,7 +542,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
542542
Y_UNIT_TEST(MySQL) {
543543
Test(
544544
NYql::EDatabaseType::MySQL,
545-
NYql::NConnector::NApi::EProtocol::NATIVE,
545+
NYql::EGenericProtocol::NATIVE,
546546
"https://mdb.api.cloud.yandex.net:443/managed-mysql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
547547
"200",
548548
R"({
@@ -590,7 +590,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
590590

591591
Test(
592592
NYql::EDatabaseType::MySQL,
593-
NYql::NConnector::NApi::EProtocol::NATIVE,
593+
NYql::EGenericProtocol::NATIVE,
594594
"https://mdb.api.cloud.yandex.net:443/managed-mysql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
595595
"403",
596596
R"(
@@ -624,7 +624,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
624624
};
625625
Test(
626626
NYql::EDatabaseType::DataStreams,
627-
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
627+
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
628628
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
629629
"403",
630630
R"(
@@ -642,7 +642,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
642642

643643
NYql::TDatabaseAuth databaseAuth;
644644
databaseAuth.UseTls = true;
645-
databaseAuth.Protocol = NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED;
645+
databaseAuth.Protocol = NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED;
646646

647647
TString databaseId1{"etn021us5r9rhld1vgb1"};
648648
TString databaseId2{"etn021us5r9rhld1vgb2"};

‎ydb/core/fq/libs/actors/ya.make

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@ PEERDIR(
7272
ydb/library/yql/providers/dq/provider
7373
ydb/library/yql/providers/dq/provider/exec
7474
ydb/library/yql/providers/dq/worker_manager/interface
75-
ydb/library/yql/providers/generic/connector/api/common
7675
ydb/library/yql/providers/generic/connector/libcpp
7776
ydb/library/yql/providers/generic/provider
7877
ydb/library/yql/providers/pq/cm_client

‎ydb/core/fq/libs/config/protos/ya.make

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ PEERDIR(
3434
ydb/library/yql/providers/s3/proto
3535
)
3636

37-
EXCLUDE_TAGS(GO_PROTO)
37+
ONLY_TAGS(
38+
CPP_PROTO
39+
PY_PROTO
40+
PY3_PROTO
41+
)
3842

3943
END()

0 commit comments

Comments
 (0)