Skip to content

Commit ddd89a2

Browse files
jepett0jepett0
authored
25-1: Ingore basic scheme limits for internal operations (#16002)
1 parent a44518d commit ddd89a2

7 files changed

+153
-16
lines changed

‎ydb/core/protos/index_builder.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ message TEvCreateRequest {
4949
optional uint64 TxId = 2;
5050
optional string DatabaseName = 3;
5151
optional TIndexBuildSettings Settings = 4;
52+
// Internal flag is true for system-generated operations and is false for those initiated directly by the user.
53+
optional bool Internal = 5 [default = false];
5254
}
5355

5456
message TEvCreateResponse {

‎ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ TVector<ISubOperation::TPtr> CreateBuildColumn(TOperationId opId, const TTxTrans
2424
{
2525
auto outTx = TransactionTemplate(table.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexMainTable);
2626
*outTx.MutableLockGuard() = tx.GetLockGuard();
27+
outTx.SetInternal(tx.GetInternal());
2728

2829
auto& snapshot = *outTx.MutableInitiateBuildIndexMainTable();
2930
snapshot.SetTableName(table.LeafName());
@@ -62,8 +63,12 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
6263
checks
6364
.IsValidLeafName()
6465
.PathsLimit(2) // index and impl-table
65-
.DirChildrenLimit()
66-
.ShardsLimit(1); // impl-table
66+
.DirChildrenLimit();
67+
68+
if (!tx.GetInternal()) {
69+
checks
70+
.ShardsLimit(1); // impl-table
71+
}
6772

6873
if (!checks) {
6974
return {CreateReject(opId, checks.GetStatus(), checks.GetError())};
@@ -95,6 +100,7 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
95100
*outTx.MutableLockGuard() = tx.GetLockGuard();
96101
outTx.MutableCreateTableIndex()->CopyFrom(indexDesc);
97102
outTx.MutableCreateTableIndex()->SetState(NKikimrSchemeOp::EIndexStateWriteOnly);
103+
outTx.SetInternal(tx.GetInternal());
98104

99105
if (!indexDesc.HasType()) {
100106
outTx.MutableCreateTableIndex()->SetType(NKikimrSchemeOp::EIndexTypeGlobal);
@@ -106,6 +112,7 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
106112
{
107113
auto outTx = TransactionTemplate(table.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexMainTable);
108114
*outTx.MutableLockGuard() = tx.GetLockGuard();
115+
outTx.SetInternal(tx.GetInternal());
109116

110117
auto& snapshot = *outTx.MutableInitiateBuildIndexMainTable();
111118
snapshot.SetTableName(table.LeafName());
@@ -118,6 +125,7 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
118125

119126
auto outTx = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexImplTable);
120127
*outTx.MutableCreateTable() = std::move(implTableDesc);
128+
outTx.SetInternal(tx.GetInternal());
121129

122130
return CreateInitializeBuildIndexImplTable(NextPartId(opId, result), outTx);
123131
};

‎ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,9 +110,13 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
110110
{
111111
auto checks = baseTablePath.Check();
112112
checks
113-
.PathShardsLimit(baseShards)
114-
.PathsLimit(pathToCreate)
115-
.ShardsLimit(shardsToCreate);
113+
.PathsLimit(pathToCreate);
114+
115+
if (!tx.GetInternal()) {
116+
checks
117+
.PathShardsLimit(baseShards)
118+
.ShardsLimit(shardsToCreate);
119+
}
116120

117121
if (!checks) {
118122
return {CreateReject(nextId, NKikimrScheme::EStatus::StatusResourceExhausted, checks.GetError())};
@@ -224,6 +228,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
224228
auto scheme = TransactionTemplate(tx.GetWorkingDir(), NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable);
225229
scheme.SetFailOnExist(tx.GetFailOnExist());
226230
scheme.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir());
231+
scheme.SetInternal(tx.GetInternal());
227232

228233
scheme.MutableCreateTable()->CopyFrom(baseTableDescription);
229234
if (tx.HasAlterUserAttributes()) {
@@ -270,6 +275,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
270275
NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable);
271276
scheme.SetFailOnExist(tx.GetFailOnExist());
272277
scheme.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir());
278+
scheme.SetInternal(tx.GetInternal());
273279

274280
*scheme.MutableCreateTable() = std::move(implTableDesc);
275281

@@ -311,6 +317,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
311317
NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence);
312318
scheme.SetFailOnExist(tx.GetFailOnExist());
313319
scheme.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir());
320+
scheme.SetInternal(tx.GetInternal());
314321

315322
*scheme.MutableSequence() = sequenceDescription;
316323

‎ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -503,9 +503,13 @@ class TCreateTable: public TSubOperation {
503503
.IsValidLeafName()
504504
.PathsLimit()
505505
.DirChildrenLimit()
506-
.ShardsLimit(shardsToCreate)
507-
.PathShardsLimit(shardsToCreate)
508506
.IsValidACL(acl);
507+
508+
if (!Transaction.GetInternal()) {
509+
checks
510+
.ShardsLimit(shardsToCreate)
511+
.PathShardsLimit(shardsToCreate);
512+
}
509513
}
510514

511515
if (!checks) {

‎ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,12 @@ class TSchemeShard::TIndexBuilder::TTxCreate: public TSchemeShard::TIndexBuilder
117117
checks
118118
.IsValidLeafName()
119119
.PathsLimit(2) // index and impl-table
120-
.DirChildrenLimit()
121-
.ShardsLimit(1); // impl-table
120+
.DirChildrenLimit();
121+
122+
if (!request.GetInternal()) {
123+
checks
124+
.ShardsLimit(1); // impl-table
125+
}
122126

123127
if (!checks) {
124128
return Reply(checks.GetStatus(), checks.GetError());

‎ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,9 @@ THolder<TEvIndexBuilder::TEvCreateRequest> BuildIndexPropose(
220220

221221
const TPath domainPath = TPath::Init(importInfo->DomainPathId, ss);
222222
auto propose = MakeHolder<TEvIndexBuilder::TEvCreateRequest>(ui64(txId), domainPath.PathString(), std::move(settings));
223-
(*propose->Record.MutableOperationParams()->mutable_labels())["uid"] = uid;
223+
auto& request = propose->Record;
224+
(*request.MutableOperationParams()->mutable_labels())["uid"] = uid;
225+
request.SetInternal(true);
224226

225227
return propose;
226228
}
@@ -261,11 +263,11 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateChangefeedPropose(
261263
if (!FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)) {
262264
return nullptr;
263265
}
264-
266+
265267
if (topic.has_retention_period()) {
266268
cdcStream.SetRetentionPeriodSeconds(topic.retention_period().seconds());
267269
}
268-
270+
269271
if (topic.has_partitioning_settings()) {
270272
i64 minActivePartitions =
271273
topic.partitioning_settings().min_active_partitions();

‎ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp

Lines changed: 114 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5388,6 +5388,116 @@ Y_UNIT_TEST_SUITE(TImportTests) {
53885388
Y_UNIT_TEST(ChangefeedsExportRestoreUnhappyPropose) {
53895389
ChangefeedsExportRestore(false);
53905390
}
5391+
5392+
Y_UNIT_TEST(IgnoreBasicSchemeLimits) {
5393+
TTestBasicRuntime runtime;
5394+
TTestEnv env(runtime);
5395+
ui64 txId = 100;
5396+
5397+
TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", R"(
5398+
Name: "Alice"
5399+
)");
5400+
TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", R"(
5401+
Name: "Alice"
5402+
ExternalSchemeShard: true
5403+
PlanResolution: 50
5404+
Coordinators: 1
5405+
Mediators: 1
5406+
TimeCastBucketsPerMediator: 2
5407+
StoragePools {
5408+
Name: "Alice:hdd"
5409+
Kind: "hdd"
5410+
}
5411+
)");
5412+
env.TestWaitNotification(runtime, txId);
5413+
5414+
ui64 tenantSchemeShard = 0;
5415+
TestDescribeResult(DescribePath(runtime, "/MyRoot/Alice"), {
5416+
NLs::ExtractTenantSchemeshard(&tenantSchemeShard)
5417+
});
5418+
UNIT_ASSERT_UNEQUAL(tenantSchemeShard, 0);
5419+
5420+
TSchemeLimits basicLimits;
5421+
basicLimits.MaxShards = 4;
5422+
basicLimits.MaxShardsInPath = 2;
5423+
SetSchemeshardSchemaLimits(runtime, basicLimits, tenantSchemeShard);
5424+
5425+
TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/Alice"), {
5426+
NLs::DomainLimitsIs(basicLimits.MaxPaths, basicLimits.MaxShards),
5427+
NLs::PathsInsideDomain(0),
5428+
NLs::ShardsInsideDomain(3)
5429+
});
5430+
5431+
TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/Alice", R"(
5432+
Name: "table1"
5433+
Columns { Name: "Key" Type: "Uint64" }
5434+
Columns { Name: "Value" Type: "Utf8" }
5435+
KeyColumnNames: ["Key"]
5436+
)");
5437+
env.TestWaitNotification(runtime, txId, tenantSchemeShard);
5438+
5439+
TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/Alice", R"(
5440+
Name: "table2"
5441+
Columns { Name: "Key" Type: "Uint64" }
5442+
Columns { Name: "Value" Type: "Utf8" }
5443+
KeyColumnNames: ["Key"]
5444+
)",
5445+
{ NKikimrScheme::StatusResourceExhausted }
5446+
);
5447+
5448+
const auto data = GenerateTestData(R"(
5449+
columns {
5450+
name: "key"
5451+
type { optional_type { item { type_id: UTF8 } } }
5452+
}
5453+
columns {
5454+
name: "value"
5455+
type { optional_type { item { type_id: UTF8 } } }
5456+
}
5457+
primary_key: "key"
5458+
partition_at_keys {
5459+
split_points {
5460+
type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } }
5461+
value { items { text_value: "b" } }
5462+
}
5463+
}
5464+
indexes {
5465+
name: "ByValue"
5466+
index_columns: "value"
5467+
global_index {}
5468+
}
5469+
)",
5470+
{{"a", 1}, {"b", 1}}
5471+
);
5472+
5473+
TPortManager portManager;
5474+
const ui16 port = portManager.GetPort();
5475+
5476+
TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port));
5477+
UNIT_ASSERT(s3Mock.Start());
5478+
5479+
const auto importId = ++txId;
5480+
TestImport(runtime, tenantSchemeShard, importId, "/MyRoot/Alice", Sprintf(R"(
5481+
ImportFromS3Settings {
5482+
endpoint: "localhost:%d"
5483+
scheme: HTTP
5484+
items {
5485+
source_prefix: ""
5486+
destination_path: "/MyRoot/Alice/ImportDir/Table"
5487+
}
5488+
}
5489+
)",
5490+
port
5491+
));
5492+
env.TestWaitNotification(runtime, importId, tenantSchemeShard);
5493+
TestGetImport(runtime, tenantSchemeShard, importId, "/MyRoot/Alice");
5494+
5495+
TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/Alice"), {
5496+
NLs::DomainLimitsIs(basicLimits.MaxPaths, basicLimits.MaxShards),
5497+
NLs::PathsInsideDomain(5),
5498+
NLs::ShardsInsideDomain(7)
5499+
});
5500+
}
53915501
}
53925502

53935503
Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
@@ -5779,7 +5889,7 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
57795889
const auto changefeedName = "update_changefeed";
57805890

57815891
schemes.emplace("", R"(
5782-
columns {
5892+
columns {
57835893
name: "key"
57845894
type { optional_type { item { type_id: UTF8 } } }
57855895
}
@@ -5846,11 +5956,11 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
58465956
}
58475957
}
58485958
)";
5849-
5959+
58505960
NAttr::TAttributes attr;
58515961
attr.emplace(NAttr::EKeys::TOPIC_DESCRIPTION, topicDesc);
58525962

5853-
schemes.emplace("/update_feed",
5963+
schemes.emplace("/update_feed",
58545964
TTypedScheme {
58555965
EPathTypeCdcStream,
58565966
changefeedDesc,
@@ -5867,7 +5977,7 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
58675977
Y_UNIT_TEST(CancelShouldSucceedOnSingleChangefeed) {
58685978
CancelShouldSucceed(GetSchemeWithChangefeed());
58695979
}
5870-
5980+
58715981
Y_UNIT_TEST(CancelShouldSucceedOnDependentView) {
58725982
CancelShouldSucceed(
58735983
{

0 commit comments

Comments
 (0)