Skip to content

Commit 6bb4b1c

Browse files
vitalyisaev2vitalyisaev2
authored
Merge from main (#12456)
1 parent db669ba commit 6bb4b1c

File tree

20 files changed

+221
-62
lines changed

20 files changed

+221
-62
lines changed

‎ydb/core/external_sources/external_source_factory.cpp

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,16 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
7171
{
7272
ToString(NYql::EDatabaseType::MsSQLServer),
7373
CreateExternalDataSource(TString{NYql::GenericProviderName}, {"BASIC"}, {"database_name", "use_tls"}, hostnamePatternsRegEx)
74-
}});
74+
},
75+
{
76+
ToString(NYql::EDatabaseType::Oracle),
77+
CreateExternalDataSource(TString{NYql::GenericProviderName}, {"BASIC"}, {"database_name", "use_tls", "service_name"}, hostnamePatternsRegEx)
78+
},
79+
{
80+
ToString(NYql::EDatabaseType::Logging),
81+
CreateExternalDataSource(TString{NYql::GenericProviderName}, {"SERVICE_ACCOUNT"}, {"folder_id"}, hostnamePatternsRegEx)
82+
}
83+
});
7584
}
7685

7786
}

‎ydb/core/fq/libs/actors/clusters_from_connections.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,16 @@ void AddClustersFromConnections(
295295
clusters.emplace(connectionName, GenericProviderName);
296296
break;
297297
}
298+
case FederatedQuery::ConnectionSetting::kLogging: {
299+
const auto& connection = conn.content().setting().logging();
300+
auto* clusterCfg = gatewaysConfig.MutableGeneric()->AddClusterMapping();
301+
clusterCfg->SetKind(NYql::NConnector::NApi::EDataSourceKind::LOGGING);
302+
clusterCfg->SetName(connectionName);
303+
clusterCfg->mutable_datasourceoptions()->insert({"folder_id", connection.folder_id()});
304+
FillClusterAuth(*clusterCfg, connection.auth(), authToken, accountIdSignatures);
305+
clusters.emplace(connectionName, GenericProviderName);
306+
break;
307+
}
298308

299309
// Do not replace with default. Adding a new connection should cause a compilation error
300310
case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET:

‎ydb/core/fq/libs/cloud_audit/yq_cloud_audit_service.cpp

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,13 @@ std::string MapConnectionType(const FederatedQuery::ConnectionSetting::Connectio
6565
return "Monitoring";
6666
case FederatedQuery::ConnectionSetting::ConnectionCase::kPostgresqlCluster:
6767
return "PostgreSQLCluster";
68-
default:
68+
case FederatedQuery::ConnectionSetting::ConnectionCase::kGreenplumCluster:
69+
return "GreenplumCluster";
70+
case FederatedQuery::ConnectionSetting::ConnectionCase::kMysqlCluster:
71+
return "MySQLCluster";
72+
case FederatedQuery::ConnectionSetting::ConnectionCase::kLogging:
73+
return "Logging";
74+
case FederatedQuery::ConnectionSetting::ConnectionCase::CONNECTION_NOT_SET:
6975
Y_ENSURE(false, "Invalid connection case " << i32(connectionCase));
7076
}
7177
}

‎ydb/core/fq/libs/common/util.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,9 @@ TString ExtractServiceAccountId(const FederatedQuery::ConnectionSetting& setting
138138
case FederatedQuery::ConnectionSetting::kMysqlCluster: {
139139
return GetServiceAccountId(setting.mysql_cluster().auth());
140140
}
141+
case FederatedQuery::ConnectionSetting::kLogging: {
142+
return GetServiceAccountId(setting.logging().auth());
143+
}
141144
// Do not replace with default. Adding a new connection should cause a compilation error
142145
case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET:
143146
break;
@@ -173,6 +176,8 @@ TMaybe<TString> GetLogin(const FederatedQuery::ConnectionSetting& setting) {
173176
return setting.greenplum_cluster().login();
174177
case FederatedQuery::ConnectionSetting::kMysqlCluster:
175178
return setting.mysql_cluster().login();
179+
case FederatedQuery::ConnectionSetting::kLogging:
180+
return {};
176181
}
177182
}
178183

@@ -196,6 +201,8 @@ TMaybe<TString> GetPassword(const FederatedQuery::ConnectionSetting& setting) {
196201
return setting.greenplum_cluster().password();
197202
case FederatedQuery::ConnectionSetting::kMysqlCluster:
198203
return setting.mysql_cluster().password();
204+
case FederatedQuery::ConnectionSetting::kLogging:
205+
return {};
199206
}
200207
}
201208

@@ -219,6 +226,8 @@ EYdbComputeAuth GetYdbComputeAuthMethod(const FederatedQuery::ConnectionSetting&
219226
return GetBasicAuthMethod(setting.greenplum_cluster().auth());
220227
case FederatedQuery::ConnectionSetting::kMysqlCluster:
221228
return GetBasicAuthMethod(setting.mysql_cluster().auth());
229+
case FederatedQuery::ConnectionSetting::kLogging:
230+
return GetIamAuthMethod(setting.logging().auth());
222231
}
223232
}
224233

@@ -240,6 +249,8 @@ FederatedQuery::IamAuth GetAuth(const FederatedQuery::Connection& connection) {
240249
return connection.content().setting().greenplum_cluster().auth();
241250
case FederatedQuery::ConnectionSetting::kMysqlCluster:
242251
return connection.content().setting().mysql_cluster().auth();
252+
case FederatedQuery::ConnectionSetting::kLogging:
253+
return connection.content().setting().logging().auth();
243254
case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET:
244255
return FederatedQuery::IamAuth{};
245256
}

‎ydb/core/fq/libs/compute/common/config.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,8 @@ class TComputeConfig {
158158
}
159159
}
160160

161+
// This function shows which external data sources are currently supported by the open-source YDB
162+
// and which ones are not yet supported.
161163
bool IsConnectionCaseEnabled(
162164
const FederatedQuery::ConnectionSetting::ConnectionCase& connectionCase) const {
163165
switch (connectionCase) {
@@ -167,6 +169,7 @@ class TComputeConfig {
167169
case FederatedQuery::ConnectionSetting::kGreenplumCluster:
168170
case FederatedQuery::ConnectionSetting::kMysqlCluster:
169171
case FederatedQuery::ConnectionSetting::kYdbDatabase:
172+
case FederatedQuery::ConnectionSetting::kLogging:
170173
return true;
171174
case FederatedQuery::ConnectionSetting::kDataStreams:
172175
case FederatedQuery::ConnectionSetting::kMonitoring:

‎ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,6 @@ TString MakeCreateExternalDataSourceQuery(
246246
"database_name"_a = EncloseAndEscapeString(connectionContent.setting().greenplum_cluster().database_name(), '"'),
247247
"use_tls"_a = common.GetDisableSslForGenericDataSources() ? "false" : "true",
248248
"schema"_a = gpschema ? ", SCHEMA=" + EncloseAndEscapeString(gpschema, '"') : TString{});
249-
250249
}
251250
break;
252251
case FederatedQuery::ConnectionSetting::kMysqlCluster: {
@@ -260,7 +259,15 @@ TString MakeCreateExternalDataSourceQuery(
260259
"mdb_cluster_id"_a = EncloseAndEscapeString(connectionContent.setting().mysql_cluster().database_id(), '"'),
261260
"database_name"_a = EncloseAndEscapeString(connectionContent.setting().mysql_cluster().database_name(), '"'),
262261
"use_tls"_a = common.GetDisableSslForGenericDataSources() ? "false" : "true");
263-
262+
}
263+
case FederatedQuery::ConnectionSetting::kLogging: {
264+
properties = fmt::format(
265+
R"(
266+
SOURCE_TYPE="Logging",
267+
FOLDER_ID={folder_id}
268+
)",
269+
"folder_id"_a = EncloseAndEscapeString(connectionContent.setting().logging().folder_id(), '"'));
270+
break;
264271
}
265272
break;
266273
}

‎ydb/core/fq/libs/control_plane_proxy/utils/utils.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ TString ExtractServiceAccountIdWithConnection(const T& setting) {
3737
case FederatedQuery::ConnectionSetting::kMysqlCluster: {
3838
return GetServiceAccountId(setting.mysql_cluster().auth());
3939
}
40+
case FederatedQuery::ConnectionSetting::kLogging: {
41+
return GetServiceAccountId(setting.logging().auth());
42+
}
4043
// Do not replace with default. Adding a new connection should cause a compilation error
4144
case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET:
4245
break;

‎ydb/core/fq/libs/control_plane_storage/request_validators.cpp

Lines changed: 20 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ void ValidateGenericConnectionSetting(
1919
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "current iam authorization is disabled"));
2020
}
2121

22-
if (!connection.database_id() && !(connection.host() && connection.port())) {
23-
auto msg = TStringBuilder() << "content.setting." << dataSourceKind << "_cluster.{database_id or host,port} field is not specified";
22+
if (!connection.database_id()) {
23+
auto msg = TStringBuilder() << "content.setting." << dataSourceKind << "_cluster.database_id field is not specified";
2424
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST,msg));
2525
}
2626

@@ -75,34 +75,11 @@ NYql::TIssues ValidateConnectionSetting(
7575
break;
7676
}
7777
case FederatedQuery::ConnectionSetting::kGreenplumCluster: {
78-
const FederatedQuery::GreenplumCluster& greenplumCluster = setting.greenplum_cluster();
79-
80-
if (!greenplumCluster.has_auth() || greenplumCluster.auth().identity_case() == FederatedQuery::IamAuth::IDENTITY_NOT_SET) {
81-
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "content.setting.greenplum_database.auth field is not specified"));
82-
}
83-
84-
if (greenplumCluster.auth().identity_case() == FederatedQuery::IamAuth::kCurrentIam && disableCurrentIam) {
85-
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "current iam authorization is disabled"));
86-
}
87-
88-
if (!greenplumCluster.database_id() && !greenplumCluster.database_name()) {
89-
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "content.setting.greenplum_database.{database_id or database_name} field is not specified"));
90-
}
78+
ValidateGenericConnectionSetting(setting.greenplum_cluster(), "greenplum", disableCurrentIam, passwordRequired, issues);
9179
break;
9280
}
9381
case FederatedQuery::ConnectionSetting::kMysqlCluster: {
94-
const FederatedQuery::MySQLCluster database = setting.mysql_cluster();
95-
if (!database.has_auth() || database.auth().identity_case() == FederatedQuery::IamAuth::IDENTITY_NOT_SET) {
96-
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "content.setting.mysql_database.auth field is not specified"));
97-
}
98-
99-
if (database.auth().identity_case() == FederatedQuery::IamAuth::kCurrentIam && disableCurrentIam) {
100-
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "current iam authorization is disabled"));
101-
}
102-
103-
if (!database.database_id() && !database.database_name()) {
104-
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "content.setting.mysql_database.{database_id or database_name} field is not specified"));
105-
}
82+
ValidateGenericConnectionSetting(setting.mysql_cluster(), "mysql", disableCurrentIam, passwordRequired, issues);
10683
break;
10784
}
10885
case FederatedQuery::ConnectionSetting::kObjectStorage: {
@@ -154,6 +131,22 @@ NYql::TIssues ValidateConnectionSetting(
154131
}
155132
break;
156133
}
134+
case FederatedQuery::ConnectionSetting::kLogging: {
135+
const FederatedQuery::Logging logging = setting.logging();
136+
if (!logging.has_auth() || logging.auth().identity_case() == FederatedQuery::IamAuth::IDENTITY_NOT_SET) {
137+
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "content.setting.logging.auth field is not specified"));
138+
}
139+
140+
if (logging.auth().identity_case() == FederatedQuery::IamAuth::kCurrentIam && disableCurrentIam) {
141+
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "current iam authorization is disabled"));
142+
}
143+
144+
if (!logging.folder_id()) {
145+
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "content.setting.logging.folder_id field is not specified"));
146+
}
147+
148+
break;
149+
}
157150
case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET: {
158151
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "connection is not set"));
159152
break;

‎ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ FederatedQuery::IamAuth::IdentityCase GetIamAuth(const FederatedQuery::Connectio
4343
return setting.greenplum_cluster().auth().identity_case();
4444
case FederatedQuery::ConnectionSetting::kMysqlCluster:
4545
return setting.mysql_cluster().auth().identity_case();
46+
case FederatedQuery::ConnectionSetting::kLogging:
47+
return setting.logging().auth().identity_case();
4648
case FederatedQuery::ConnectionSetting::CONNECTION_NOT_SET:
4749
return FederatedQuery::IamAuth::IDENTITY_NOT_SET;
4850
}

‎ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,10 +72,12 @@ void FillCreateExternalDataSourceDesc(NKikimrSchemeOp::TExternalDataSourceDescri
7272
static const TSet<TString> properties {
7373
"database_name",
7474
"protocol", // managed PG, CH
75-
"mdb_cluster_id", // managed PG, CH
75+
"mdb_cluster_id", // managed PG, CH, GP, MY
7676
"database_id", // managed YDB
7777
"use_tls",
78-
"schema", // managed PG
78+
"schema", // managed PG, GP
79+
"service_name", // oracle
80+
"folder_id" // logging
7981
};
8082

8183
for (const auto& property: properties) {

0 commit comments

Comments
 (0)