Skip to content

Commit 97e3d04

Browse files
kardymondskardymonds
authored
YQ-4084 Available external data sources / to 25.1-analytics (#17984)
2 parents 3c5ebc4 + 7343abb commit 97e3d04

File tree

34 files changed

+426
-142
lines changed

34 files changed

+426
-142
lines changed

‎ydb/core/external_sources/external_source_factory.cpp

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,30 @@ namespace NKikimr::NExternalSource {
1414
namespace {
1515

1616
struct TExternalSourceFactory : public IExternalSourceFactory {
17-
TExternalSourceFactory(const TMap<TString, IExternalSource::TPtr>& sources)
17+
TExternalSourceFactory(
18+
const TMap<TString, IExternalSource::TPtr>& sources,
19+
bool allExternalDataSourcesAreAvailable,
20+
const std::set<TString>& availableExternalDataSources)
1821
: Sources(sources)
22+
, AllExternalDataSourcesAreAvailable(allExternalDataSourcesAreAvailable)
23+
, AvailableExternalDataSources(availableExternalDataSources)
1924
{}
2025

2126
IExternalSource::TPtr GetOrCreate(const TString& type) const override {
2227
auto it = Sources.find(type);
23-
if (it != Sources.end()) {
24-
return it->second;
28+
if (it == Sources.end()) {
29+
throw TExternalSourceException() << "External source with type " << type << " was not found";
2530
}
26-
throw TExternalSourceException() << "External source with type " << type << " was not found";
31+
if (!AllExternalDataSourcesAreAvailable && !AvailableExternalDataSources.contains(type)) {
32+
throw TExternalSourceException() << "External source with type " << type << " is disabled. Please contact your system administrator to enable it";
33+
}
34+
return it->second;
2735
}
2836

2937
private:
30-
TMap<TString, IExternalSource::TPtr> Sources;
38+
const TMap<TString, IExternalSource::TPtr> Sources;
39+
bool AllExternalDataSourcesAreAvailable;
40+
const std::set<TString> AvailableExternalDataSources;
3141
};
3242

3343
}
@@ -37,7 +47,9 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
3747
size_t pathsLimit,
3848
std::shared_ptr<NYql::ISecuredServiceAccountCredentialsFactory> credentialsFactory,
3949
bool enableInfer,
40-
bool allowLocalFiles) {
50+
bool allowLocalFiles,
51+
bool allExternalDataSourcesAreAvailable,
52+
const std::set<TString>& availableExternalDataSources) {
4153
std::vector<TRegExMatch> hostnamePatternsRegEx(hostnamePatterns.begin(), hostnamePatterns.end());
4254
return MakeIntrusive<TExternalSourceFactory>(TMap<TString, IExternalSource::TPtr>{
4355
{
@@ -84,7 +96,9 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
8496
ToString(NYql::EDatabaseType::Solomon),
8597
CreateExternalDataSource(TString{NYql::SolomonProviderName}, {"NONE", "TOKEN"}, {"use_ssl", "grpc_port"}, hostnamePatternsRegEx)
8698
}
87-
});
99+
},
100+
allExternalDataSourcesAreAvailable,
101+
availableExternalDataSources);
88102
}
89103

90104
}

‎ydb/core/external_sources/external_source_factory.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
#include "external_source.h"
44
#include <ydb/library/yql/providers/common/token_accessor/client/factory.h>
5+
#include <ydb/library/yql/providers/common/db_id_async_resolver/database_type.h>
56

67
namespace NKikimr::NExternalSource {
78

@@ -16,6 +17,8 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
1617
size_t pathsLimit = 50000,
1718
std::shared_ptr<NYql::ISecuredServiceAccountCredentialsFactory> credentialsFactory = nullptr,
1819
bool enableInfer = false,
19-
bool allowLocalFiles = false);
20+
bool allowLocalFiles = false,
21+
bool allExternalDataSourcesAreAvailable = true,
22+
const std::set<TString>& availableExternalDataSources = {});
2023

2124
}

‎ydb/core/grpc_services/rpc_describe_external_table.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ bool ConvertContent(
3636
google::protobuf::Map<TProtoStringType, TProtoStringType>& out,
3737
TIssues& issues
3838
) {
39-
const auto externalSourceFactory = NExternalSource::CreateExternalSourceFactory({});
39+
const auto externalSourceFactory = NExternalSource::CreateExternalSourceFactory({}, nullptr, 50000, nullptr, false, false, true, NYql::GetAllExternalDataSourceTypes());
4040
try {
4141
const auto source = externalSourceFactory->GetOrCreate(sourceType);
4242
for (const auto& [key, items] : source->GetParameters(in)) {

‎ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "kqp_federated_query_helpers.h"
22

33
#include <ydb/library/actors/http/http_proxy.h>
4+
#include <ydb/library/yql/providers/common/db_id_async_resolver/database_type.h>
45

56
#include <ydb/core/base/counters.h>
67
#include <ydb/core/base/feature_flags.h>
@@ -57,6 +58,11 @@ namespace NKikimr::NKqp {
5758
return std::make_pair(ToString(host), scheme == "grpcs");
5859
}
5960

61+
bool IsValidExternalDataSourceType(const TString& type) {
62+
static auto allTypes = NYql::GetAllExternalDataSourceTypes();
63+
return allTypes.contains(type);
64+
}
65+
6066
// TKqpFederatedQuerySetupFactoryDefault contains network clients and service actors necessary
6167
// for federated queries. HTTP Gateway (required by S3 provider) is run by default even without
6268
// explicit configuration. Token Accessor and Connector Client are run only if config is provided.
@@ -160,6 +166,11 @@ namespace NKikimr::NKqp {
160166
return std::make_shared<TKqpFederatedQuerySetupFactoryNoop>();
161167
}
162168

169+
for (const auto& source : appConfig.GetQueryServiceConfig().GetAvailableExternalDataSources()) {
170+
if (!IsValidExternalDataSourceType(source)) {
171+
ythrow yexception() << "wrong AvailableExternalDataSources \"" << source << "\"";
172+
}
173+
}
163174
return std::make_shared<NKikimr::NKqp::TKqpFederatedQuerySetupFactoryDefault>(setup, appData, appConfig);
164175
}
165176

‎ydb/core/kqp/gateway/kqp_metadata_loader.cpp

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -869,7 +869,14 @@ NThreading::TFuture<TTableMetadataResult> TKqpTableMetadataLoader::LoadTableMeta
869869

870870
NExternalSource::IExternalSource::TPtr externalSource;
871871
if (settings.ExternalSourceFactory) {
872-
externalSource = settings.ExternalSourceFactory->GetOrCreate(externalDataSourceMetadata.Metadata->ExternalSource.Type);
872+
try {
873+
externalSource = settings.ExternalSourceFactory->GetOrCreate(externalDataSourceMetadata.Metadata->ExternalSource.Type);
874+
} catch (const std::exception& exception) {
875+
TTableMetadataResult wrapper;
876+
wrapper.SetException(yexception() << "couldn't get external source with type " << externalDataSourceMetadata.Metadata->ExternalSource.Type << ", " << exception.what());
877+
promise.SetValue(wrapper);
878+
return;
879+
}
873880
}
874881

875882
if (externalSource && externalSource->CanLoadDynamicMetadata()) {

‎ydb/core/kqp/host/kqp_host.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1164,12 +1164,16 @@ class TKqpHost : public IKqpHost {
11641164
}
11651165

11661166
if (FederatedQuerySetup) {
1167-
ExternalSourceFactory = NExternalSource::CreateExternalSourceFactory({},
1167+
const auto& hostnamePatterns = QueryServiceConfig.GetHostnamePatterns();
1168+
const auto& availableExternalDataSources = QueryServiceConfig.GetAvailableExternalDataSources();
1169+
ExternalSourceFactory = NExternalSource::CreateExternalSourceFactory(std::vector<TString>(hostnamePatterns.begin(), hostnamePatterns.end()),
11681170
ActorSystem,
11691171
FederatedQuerySetup->S3GatewayConfig.GetGeneratorPathsLimit(),
11701172
FederatedQuerySetup ? FederatedQuerySetup->CredentialsFactory : nullptr,
11711173
Config->FeatureFlags.GetEnableExternalSourceSchemaInference(),
1172-
FederatedQuerySetup->S3GatewayConfig.GetAllowLocalFiles());
1174+
FederatedQuerySetup->S3GatewayConfig.GetAllowLocalFiles(),
1175+
QueryServiceConfig.GetAllExternalDataSourcesAreAvailable(),
1176+
std::set<TString>(availableExternalDataSources.cbegin(), availableExternalDataSources.cend()));
11731177
}
11741178
}
11751179

‎ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -296,37 +296,47 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
296296
}
297297

298298
Y_UNIT_TEST(TestCreateExternalTable) {
299-
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false));
299+
NKikimrConfig::TAppConfig appCfg;
300+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
301+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false).SetAppConfig(appCfg));
300302
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
301303
TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
302304
TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table");
303305
}
304306

305307
Y_UNIT_TEST(TestCreateSameExternalTable) {
306-
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false));
308+
NKikimrConfig::TAppConfig appCfg;
309+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
310+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false).SetAppConfig(appCfg));
307311
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
308312
TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
309313
TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table");
310314
TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table", true);
311315
}
312316

313317
Y_UNIT_TEST(TestDropExternalTable) {
314-
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false));
318+
NKikimrConfig::TAppConfig appCfg;
319+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
320+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false).SetAppConfig(appCfg));
315321
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
316322
TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
317323
TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table");
318324
TestDropExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table");
319325
}
320326

321327
Y_UNIT_TEST(TestDropExternalDataSource) {
322-
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false));
328+
NKikimrConfig::TAppConfig appCfg;
329+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
330+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false).SetAppConfig(appCfg));
323331
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
324332
TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
325333
TestDropExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
326334
}
327335

328336
Y_UNIT_TEST(TestLoadExternalTable) {
329-
TKikimrRunner kikimr;
337+
NKikimrConfig::TAppConfig appCfg;
338+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
339+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
330340
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
331341
auto db = kikimr.GetTableClient();
332342
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -366,7 +376,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
366376
}
367377

368378
Y_UNIT_TEST(TestLoadServiceAccountSecretValueFromExternalDataSourceMetadata) {
369-
TKikimrRunner kikimr;
379+
NKikimrConfig::TAppConfig appCfg;
380+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
381+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
370382
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
371383
auto db = kikimr.GetTableClient();
372384
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -404,7 +416,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
404416
}
405417

406418
Y_UNIT_TEST(TestLoadBasicSecretValueFromExternalDataSourceMetadata) {
407-
TKikimrRunner kikimr;
419+
NKikimrConfig::TAppConfig appCfg;
420+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("PostgreSQL");
421+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
408422
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
409423
auto db = kikimr.GetTableClient();
410424
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -435,7 +449,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
435449
}
436450

437451
Y_UNIT_TEST(TestLoadMdbBasicSecretValueFromExternalDataSourceMetadata) {
438-
TKikimrRunner kikimr;
452+
NKikimrConfig::TAppConfig appCfg;
453+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("PostgreSQL");
454+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
439455
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
440456
auto db = kikimr.GetTableClient();
441457
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -473,7 +489,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
473489
}
474490

475491
Y_UNIT_TEST(TestLoadAwsSecretValueFromExternalDataSourceMetadata) {
476-
TKikimrRunner kikimr;
492+
NKikimrConfig::TAppConfig appCfg;
493+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
494+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
477495
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
478496
auto db = kikimr.GetTableClient();
479497
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -510,7 +528,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
510528
}
511529

512530
Y_UNIT_TEST(TestLoadDataSourceProperties) {
513-
TKikimrRunner kikimr;
531+
NKikimrConfig::TAppConfig appCfg;
532+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("PostgreSQL");
533+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
514534
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
515535
auto db = kikimr.GetTableClient();
516536
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -558,7 +578,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
558578
}
559579

560580
Y_UNIT_TEST(TestLoadTokenSecretValueFromExternalDataSourceMetadata) {
561-
TKikimrRunner kikimr;
581+
NKikimrConfig::TAppConfig appCfg;
582+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("YT");
583+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
562584
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
563585
auto db = kikimr.GetTableClient();
564586
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -588,7 +610,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
588610
}
589611

590612
Y_UNIT_TEST(TestSecretsExistingValidation) {
591-
TKikimrRunner kikimr;
613+
NKikimrConfig::TAppConfig appCfg;
614+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
615+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
592616
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
593617
auto db = kikimr.GetTableClient();
594618
auto session = db.CreateSession().GetValueSync().GetSession();

‎ydb/core/kqp/ut/common/columnshard.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ namespace NKqp {
2121
}
2222
if (!kikimrSettings.FeatureFlags.HasEnableExternalDataSources()) {
2323
kikimrSettings.SetEnableExternalDataSources(true);
24+
kikimrSettings.AppConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
2425
}
2526

2627
Kikimr = std::make_unique<TKikimrRunner>(kikimrSettings);

‎ydb/core/kqp/ut/federated_query/common/common.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ namespace NKikimr::NKqp::NFederatedQueryTest {
4141
if (!appConfig) {
4242
appConfig.emplace();
4343
}
44+
appConfig->MutableQueryServiceConfig()->SetAllExternalDataSourcesAreAvailable(true);
4445

4546
auto settings = TKikimrSettings();
4647

‎ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,11 @@ namespace NKikimr::NKqp {
6969
appConfig.MutableQueryServiceConfig()->MutableGeneric()->MutableConnector()->MutableEndpoint()->set_host("localhost");
7070
appConfig.MutableQueryServiceConfig()->MutableGeneric()->MutableConnector()->MutableEndpoint()->set_port(1234);
7171
appConfig.MutableQueryServiceConfig()->MutableGeneric()->MutableDefaultSettings()->Add(std::move(dateTimeFormat));
72+
appConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
73+
appConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ClickHouse");
74+
appConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("PostgreSQL");
75+
appConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("MySQL");
76+
appConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("Ydb");
7277
return appConfig;
7378
}
7479

0 commit comments

Comments
 (0)