Skip to content

Commit 376860c

Browse files
kardymondskardymonds
committed
YQ-4084 Update default availableExternalDataSources (without query_service_config) (#15804)
1 parent d2e1345 commit 376860c

File tree

12 files changed

+125
-44
lines changed

12 files changed

+125
-44
lines changed

‎ydb/core/external_sources/external_source_factory.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,6 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
1818
std::shared_ptr<NYql::ISecuredServiceAccountCredentialsFactory> credentialsFactory = nullptr,
1919
bool enableInfer = false,
2020
bool allowLocalFiles = false,
21-
const std::set<TString>& availableExternalDataSources = NYql::GetAllExternalDataSourceTypes());
21+
const std::set<TString>& availableExternalDataSources = {});
2222

2323
}

‎ydb/core/grpc_services/rpc_describe_external_table.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ bool ConvertContent(
3636
google::protobuf::Map<TProtoStringType, TProtoStringType>& out,
3737
TIssues& issues
3838
) {
39-
const auto externalSourceFactory = NExternalSource::CreateExternalSourceFactory({});
39+
const auto externalSourceFactory = NExternalSource::CreateExternalSourceFactory({}, nullptr, 50000, nullptr, false, false, NYql::GetAllExternalDataSourceTypes());
4040
try {
4141
const auto source = externalSourceFactory->GetOrCreate(sourceType);
4242
for (const auto& [key, items] : source->GetParameters(in)) {

‎ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6642,7 +6642,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
66426642
}
66436643

66446644
Y_UNIT_TEST(CreateExternalDataSourceWithSa) {
6645-
TKikimrRunner kikimr;
6645+
NKikimrConfig::TAppConfig config;
6646+
config.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
6647+
config.MutableQueryServiceConfig()->MutableS3()->SetGeneratorPathsLimit(50000);
6648+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(config));
6649+
66466650
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
66476651
auto db = kikimr.GetTableClient();
66486652
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -6791,7 +6795,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
67916795
}
67926796

67936797
Y_UNIT_TEST(DropExternalDataSource) {
6794-
TKikimrRunner kikimr;
6798+
NKikimrConfig::TAppConfig config;
6799+
config.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
6800+
config.MutableQueryServiceConfig()->MutableS3()->SetGeneratorPathsLimit(50000);
6801+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(config));
6802+
67956803
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
67966804
auto db = kikimr.GetTableClient();
67976805
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -6831,7 +6839,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
68316839
}
68326840

68336841
Y_UNIT_TEST(DoubleCreateExternalDataSource) {
6834-
TKikimrRunner kikimr;
6842+
NKikimrConfig::TAppConfig config;
6843+
config.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
6844+
config.MutableQueryServiceConfig()->MutableS3()->SetGeneratorPathsLimit(50000);
6845+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(config));
6846+
68356847
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
68366848
auto db = kikimr.GetTableClient();
68376849
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -6872,7 +6884,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
68726884
}
68736885

68746886
Y_UNIT_TEST(CreateExternalTable) {
6875-
TKikimrRunner kikimr;
6887+
NKikimrConfig::TAppConfig config;
6888+
config.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
6889+
config.MutableQueryServiceConfig()->MutableS3()->SetGeneratorPathsLimit(50000);
6890+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(config));
6891+
68766892
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
68776893
auto db = kikimr.GetTableClient();
68786894
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -6959,7 +6975,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
69596975
}
69606976

69616977
Y_UNIT_TEST(DropExternalTable) {
6962-
TKikimrRunner kikimr;
6978+
NKikimrConfig::TAppConfig config;
6979+
config.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
6980+
config.MutableQueryServiceConfig()->MutableS3()->SetGeneratorPathsLimit(50000);
6981+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(config));
6982+
69636983
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
69646984
auto db = kikimr.GetTableClient();
69656985
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -7016,7 +7036,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
70167036
}
70177037

70187038
Y_UNIT_TEST(CreateExternalTableWithSettings) {
7019-
TKikimrRunner kikimr;
7039+
NKikimrConfig::TAppConfig config;
7040+
config.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
7041+
config.MutableQueryServiceConfig()->MutableS3()->SetGeneratorPathsLimit(50000);
7042+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(config));
7043+
70207044
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
70217045
auto db = kikimr.GetTableClient();
70227046
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -7064,7 +7088,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
70647088
}
70657089

70667090
Y_UNIT_TEST(CreateExternalTableWithUpperCaseSettings) {
7067-
TKikimrRunner kikimr;
7091+
NKikimrConfig::TAppConfig config;
7092+
config.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
7093+
config.MutableQueryServiceConfig()->MutableS3()->SetGeneratorPathsLimit(50000);
7094+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(config));
7095+
70687096
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
70697097
auto db = kikimr.GetTableClient();
70707098
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -7112,7 +7140,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
71127140
}
71137141

71147142
Y_UNIT_TEST(DoubleCreateExternalTable) {
7115-
TKikimrRunner kikimr;
7143+
NKikimrConfig::TAppConfig config;
7144+
config.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
7145+
config.MutableQueryServiceConfig()->MutableS3()->SetGeneratorPathsLimit(50000);
7146+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(config));
7147+
71167148
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
71177149
auto db = kikimr.GetTableClient();
71187150
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -7162,7 +7194,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
71627194
}
71637195

71647196
Y_UNIT_TEST(DropDependentExternalDataSource) {
7165-
TKikimrRunner kikimr;
7197+
NKikimrConfig::TAppConfig config;
7198+
config.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
7199+
config.MutableQueryServiceConfig()->MutableS3()->SetGeneratorPathsLimit(50000);
7200+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(config));
7201+
71667202
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
71677203
auto db = kikimr.GetTableClient();
71687204
auto session = db.CreateSession().GetValueSync().GetSession();

‎ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ using namespace NSchemeShardUT_Private;
88
Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
99
Y_UNIT_TEST(CreateExternalDataSource) {
1010
TTestBasicRuntime runtime;
11-
TTestEnv env(runtime);
11+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
1212
ui64 txId = 100;
1313

1414
TestCreateExternalDataSource(runtime, txId++, "/MyRoot",R"(
@@ -28,7 +28,7 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
2828

2929
Y_UNIT_TEST(CreateExternalDataSourceWithProperties) {
3030
TTestBasicRuntime runtime;
31-
TTestEnv env(runtime);
31+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
3232
ui64 txId = 100;
3333

3434
TestCreateExternalDataSource(runtime, txId++, "/MyRoot",R"(
@@ -60,7 +60,7 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
6060

6161
Y_UNIT_TEST(DropExternalDataSource) {
6262
TTestBasicRuntime runtime;
63-
TTestEnv env(runtime);
63+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
6464
ui64 txId = 100;
6565

6666
TestCreateExternalDataSource(runtime, txId++, "/MyRoot",R"(
@@ -87,7 +87,7 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
8787

8888
void DropTwice(const TString& path, TRuntimeTxFn createFn, TRuntimeTxFn dropFn) {
8989
TTestBasicRuntime runtime;
90-
TTestEnv env(runtime);
90+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
9191
ui64 txId = 100;
9292

9393
createFn(runtime, ++txId);
@@ -133,7 +133,7 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
133133

134134
Y_UNIT_TEST(ParallelCreateExternalDataSource) {
135135
TTestBasicRuntime runtime;
136-
TTestEnv env(runtime);
136+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
137137
ui64 txId = 123;
138138

139139
AsyncMkDir(runtime, ++txId, "/MyRoot", "DirA");
@@ -176,7 +176,7 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
176176
using ESts = NKikimrScheme::EStatus;
177177

178178
TTestBasicRuntime runtime;
179-
TTestEnv env(runtime);
179+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
180180
ui64 txId = 123;
181181

182182
TString dataSourceConfig = R"(
@@ -226,7 +226,7 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
226226

227227
Y_UNIT_TEST(ReadOnlyMode) {
228228
TTestBasicRuntime runtime;
229-
TTestEnv env(runtime);
229+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
230230
ui64 txId = 123;
231231

232232
AsyncMkDir(runtime, ++txId, "/MyRoot", "SubDirA");
@@ -277,7 +277,7 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
277277

278278
Y_UNIT_TEST(SchemeErrors) {
279279
TTestBasicRuntime runtime;
280-
TTestEnv env(runtime);
280+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
281281
ui64 txId = 123;
282282

283283
TestMkDir(runtime, ++txId, "/MyRoot", "DirA");
@@ -328,7 +328,7 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
328328

329329
Y_UNIT_TEST(PreventDeletionOfDependentDataSources) {
330330
TTestBasicRuntime runtime;
331-
TTestEnv env(runtime);
331+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
332332
ui64 txId = 100;
333333

334334
TestCreateExternalDataSource(runtime, txId++, "/MyRoot",R"(
@@ -367,7 +367,7 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
367367

368368
Y_UNIT_TEST(RemovingReferencesFromDataSources) {
369369
TTestBasicRuntime runtime;
370-
TTestEnv env(runtime);
370+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
371371
ui64 txId = 100;
372372

373373
TestCreateExternalDataSource(runtime, txId++, "/MyRoot",R"(
@@ -410,7 +410,7 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
410410

411411
Y_UNIT_TEST(ReplaceExternalDataSourceIfNotExists) {
412412
TTestBasicRuntime runtime;
413-
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(true));
413+
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(true).RunFakeConfigDispatcher(true));
414414
ui64 txId = 100;
415415

416416
TestCreateExternalDataSource(runtime, ++txId, "/MyRoot",R"(
@@ -465,7 +465,7 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
465465

466466
Y_UNIT_TEST(CreateExternalDataSourceShouldFailIfSuchEntityAlreadyExists) {
467467
TTestBasicRuntime runtime;
468-
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(true));
468+
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(true).RunFakeConfigDispatcher(true));
469469
ui64 txId = 100;
470470

471471
TestCreateExternalDataSource(runtime, ++txId, "/MyRoot",R"(
@@ -546,7 +546,7 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
546546

547547
Y_UNIT_TEST(ReplaceExternalDataSourceIfNotExistsShouldFailIfFeatureFlagIsNotSet) {
548548
TTestBasicRuntime runtime;
549-
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(false));
549+
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(false).RunFakeConfigDispatcher(true));
550550
ui64 txId = 100;
551551

552552
TestCreateExternalDataSource(runtime, ++txId, "/MyRoot",R"(

‎ydb/core/tx/schemeshard/ut_external_table/ut_external_table.cpp

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ void CreateExternalDataSource(TTestBasicRuntime& runtime, TTestEnv& env, ui64 tx
2828
Y_UNIT_TEST_SUITE(TExternalTableTest) {
2929
Y_UNIT_TEST(CreateExternalTable) {
3030
TTestBasicRuntime runtime;
31-
TTestEnv env(runtime);
31+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
3232
ui64 txId = 100;
3333
CreateExternalDataSource(runtime, env, txId++);
3434
TestCreateExternalTable(runtime, txId++, "/MyRoot", R"(
@@ -46,7 +46,7 @@ Y_UNIT_TEST_SUITE(TExternalTableTest) {
4646

4747
Y_UNIT_TEST(DropExternalTable) {
4848
TTestBasicRuntime runtime;
49-
TTestEnv env(runtime);
49+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
5050
ui64 txId = 100;
5151

5252
CreateExternalDataSource(runtime, env, txId++);
@@ -72,7 +72,7 @@ Y_UNIT_TEST_SUITE(TExternalTableTest) {
7272

7373
void DropTwice(const TString& path, TRuntimeTxFn createFn, TRuntimeTxFn dropFn) {
7474
TTestBasicRuntime runtime;
75-
TTestEnv env(runtime);
75+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
7676
ui64 txId = 100;
7777

7878
CreateExternalDataSource(runtime, env, txId++);
@@ -118,7 +118,7 @@ Y_UNIT_TEST_SUITE(TExternalTableTest) {
118118

119119
Y_UNIT_TEST(ParallelCreateExternalTable) {
120120
TTestBasicRuntime runtime;
121-
TTestEnv env(runtime);
121+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
122122
ui64 txId = 123;
123123

124124
CreateExternalDataSource(runtime, env, txId++);
@@ -162,7 +162,7 @@ Y_UNIT_TEST_SUITE(TExternalTableTest) {
162162
using ESts = NKikimrScheme::EStatus;
163163

164164
TTestBasicRuntime runtime;
165-
TTestEnv env(runtime);
165+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
166166
ui64 txId = 123;
167167

168168
TString tableConfig = R"(
@@ -211,7 +211,7 @@ Y_UNIT_TEST_SUITE(TExternalTableTest) {
211211

212212
Y_UNIT_TEST(ReadOnlyMode) {
213213
TTestBasicRuntime runtime;
214-
TTestEnv env(runtime);
214+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
215215
ui64 txId = 123;
216216

217217
CreateExternalDataSource(runtime, env, txId++);
@@ -261,7 +261,7 @@ Y_UNIT_TEST_SUITE(TExternalTableTest) {
261261

262262
Y_UNIT_TEST(SchemeErrors) {
263263
TTestBasicRuntime runtime;
264-
TTestEnv env(runtime);
264+
TTestEnv env(runtime, TTestEnvOptions().RunFakeConfigDispatcher(true));
265265
ui64 txId = 123;
266266

267267
TestMkDir(runtime, ++txId, "/MyRoot", "DirA");
@@ -315,7 +315,7 @@ Y_UNIT_TEST_SUITE(TExternalTableTest) {
315315

316316
Y_UNIT_TEST(ReplaceExternalTableIfNotExists) {
317317
TTestBasicRuntime runtime;
318-
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(true));
318+
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(true).RunFakeConfigDispatcher(true));
319319
ui64 txId = 100;
320320

321321
CreateExternalDataSource(runtime, env, ++txId);
@@ -381,7 +381,7 @@ Y_UNIT_TEST_SUITE(TExternalTableTest) {
381381

382382
Y_UNIT_TEST(CreateExternalTableShouldFailIfSuchEntityAlreadyExists) {
383383
TTestBasicRuntime runtime;
384-
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(true));
384+
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(true).RunFakeConfigDispatcher(true));
385385
ui64 txId = 100;
386386

387387
CreateExternalDataSource(runtime, env, ++txId);
@@ -442,7 +442,7 @@ Y_UNIT_TEST_SUITE(TExternalTableTest) {
442442

443443
Y_UNIT_TEST(ReplaceExternalTableShouldFailIfEntityOfAnotherTypeWithSameNameExists) {
444444
TTestBasicRuntime runtime;
445-
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(true));
445+
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(true).RunFakeConfigDispatcher(true));
446446
ui64 txId = 100;
447447

448448
TestCreateView(runtime, ++txId, "/MyRoot", R"(
@@ -469,7 +469,7 @@ Y_UNIT_TEST_SUITE(TExternalTableTest) {
469469

470470
Y_UNIT_TEST(ReplaceExternalTableIfNotExistsShouldFailIfFeatureFlagIsNotSet) {
471471
TTestBasicRuntime runtime;
472-
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(false));
472+
TTestEnv env(runtime, TTestEnvOptions().EnableReplaceIfExistsForExternalEntities(false).RunFakeConfigDispatcher(true));
473473
ui64 txId = 100;
474474

475475
CreateExternalDataSource(runtime, env, ++txId);
@@ -489,7 +489,7 @@ Y_UNIT_TEST_SUITE(TExternalTableTest) {
489489

490490
Y_UNIT_TEST(Decimal) {
491491
TTestBasicRuntime runtime;
492-
TTestEnv env(runtime, TTestEnvOptions().EnableParameterizedDecimal(true));
492+
TTestEnv env(runtime, TTestEnvOptions().EnableParameterizedDecimal(true).RunFakeConfigDispatcher(true));
493493
ui64 txId = 100;
494494
CreateExternalDataSource(runtime, env, txId++);
495495
TestCreateExternalTable(runtime, txId++, "/MyRoot", R"_(

0 commit comments

Comments
 (0)