Skip to content

Commit 79c834b

Browse files
jepett0jepett0
authored
Add a feature flag controlling separate disk space quotas accounting (#10936)
1 parent c850697 commit 79c834b

File tree

5 files changed

+176
-92
lines changed

5 files changed

+176
-92
lines changed

‎ydb/core/protos/feature_flags.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,4 +175,5 @@ message TFeatureFlags {
175175
optional bool EnableFollowerStats = 150 [default = true];
176176
optional bool EnableTopicAutopartitioningForReplication = 151 [default = false];
177177
optional bool EnableDriveSerialsDiscovery = 152 [default = false];
178+
optional bool EnableSeparateDiskSpaceQuotas = 153 [default = false];
178179
}

‎ydb/core/tx/schemeshard/schemeshard_info_types.cpp

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ EDiskUsageStatus CheckStoragePoolsQuotas(const THashMap<TString, TStoragePoolUsa
3737
for (const auto& [poolKind, usage] : storagePoolsUsage) {
3838
if (const auto* quota = storagePoolsQuotas.FindPtr(poolKind)) {
3939
const auto totalSize = usage.DataSize + usage.IndexSize;
40+
// If a quota is equal to zero, then it sets no limit on the disk space usage.
4041
if (quota->HardQuota && totalSize > quota->HardQuota) {
4142
return EDiskUsageStatus::AboveHardQuota;
4243
}
@@ -152,24 +153,40 @@ bool TSubDomainInfo::CheckDiskSpaceQuotas(IQuotaCounters* counters) {
152153
return changeSubdomainState(EDiskUsageStatus::BelowSoftQuota);
153154
}
154155

155-
ui64 totalUsage = TotalDiskSpaceUsage();
156-
const auto storagePoolsUsageStatus = CheckStoragePoolsQuotas(DiskSpaceUsage.StoragePoolsUsage, quotas.StoragePoolsQuotas);
156+
if (!AppData()->FeatureFlags.GetEnableSeparateDiskSpaceQuotas()) {
157+
// If the feature flag is turned off, then the storage pool quotas are ignored:
158+
// they are not accounted for when we make a decision to change the state of the subdomain.
159+
ui64 totalUsage = TotalDiskSpaceUsage();
157160

158-
// Quota being equal to zero is treated as if there is no limit set on disk space usage.
159-
const bool overallHardQuotaIsExceeded = quotas.HardQuota && totalUsage > quotas.HardQuota;
160-
const bool someStoragePoolHardQuotaIsExceeded = !quotas.StoragePoolsQuotas.empty()
161-
&& storagePoolsUsageStatus == EDiskUsageStatus::AboveHardQuota;
162-
if (overallHardQuotaIsExceeded || someStoragePoolHardQuotaIsExceeded) {
163-
return changeSubdomainState(EDiskUsageStatus::AboveHardQuota);
164-
}
161+
// If a quota is equal to zero, then it sets no limit on the disk space usage.
162+
const bool isHardQuotaExceeded = quotas.HardQuota && totalUsage > quotas.HardQuota;
163+
const bool isTotalUsageBelowSoftQuota = !quotas.SoftQuota || totalUsage < quotas.SoftQuota;
165164

166-
const bool totalUsageIsBelowOverallSoftQuota = !quotas.SoftQuota || totalUsage < quotas.SoftQuota;
167-
const bool allStoragePoolsUsageIsBelowSoftQuota = quotas.StoragePoolsQuotas.empty()
168-
|| storagePoolsUsageStatus == EDiskUsageStatus::BelowSoftQuota;
169-
if (totalUsageIsBelowOverallSoftQuota && allStoragePoolsUsageIsBelowSoftQuota) {
170-
return changeSubdomainState(EDiskUsageStatus::BelowSoftQuota);
165+
if (isHardQuotaExceeded) {
166+
return changeSubdomainState(EDiskUsageStatus::AboveHardQuota);
167+
}
168+
if (isTotalUsageBelowSoftQuota) {
169+
return changeSubdomainState(EDiskUsageStatus::BelowSoftQuota);
170+
}
171+
} else {
172+
// If the feature flag is turned on, then the overall quota is ignored:
173+
// it is not accounted for when we make a decision to change the state of the subdomain.
174+
const auto storagePoolsUsageStatus = CheckStoragePoolsQuotas(DiskSpaceUsage.StoragePoolsUsage, quotas.StoragePoolsQuotas);
175+
176+
const bool isSomeStoragePoolHardQuotaExceeded = !quotas.StoragePoolsQuotas.empty()
177+
&& storagePoolsUsageStatus == EDiskUsageStatus::AboveHardQuota;
178+
const bool isEachStoragePoolUsageBelowSoftQuota = quotas.StoragePoolsQuotas.empty()
179+
|| storagePoolsUsageStatus == EDiskUsageStatus::BelowSoftQuota;
180+
181+
if (isSomeStoragePoolHardQuotaExceeded) {
182+
return changeSubdomainState(EDiskUsageStatus::AboveHardQuota);
183+
}
184+
if (isEachStoragePoolUsageBelowSoftQuota) {
185+
return changeSubdomainState(EDiskUsageStatus::BelowSoftQuota);
186+
}
171187
}
172188

189+
// made no changes to the state of the subdomain
173190
return false;
174191
}
175192

‎ydb/core/tx/schemeshard/ut_subdomain/ut_subdomain.cpp

Lines changed: 64 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ constexpr const char* EntireDatabaseTag = "entire_database";
130130

131131
NLs::TCheckFunc LsCheckDiskQuotaExceeded(
132132
const TMap<TString, EDiskUsageStatus>& expectedExceeders,
133+
bool enableSeparateQuotasFeatureFlag,
133134
const TString& debugHint = ""
134135
) {
135136
return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) {
@@ -166,8 +167,19 @@ NLs::TCheckFunc LsCheckDiskQuotaExceeded(
166167
receivedQuotas.data_size_soft_quota()
167168
}
168169
);
169-
170+
170171
TMap<TString, EDiskUsageStatus> exceeders = CheckStoragePoolsQuotas(parsedUsage, parsedQuotas);
172+
if (enableSeparateQuotasFeatureFlag) {
173+
// ignore the status of the overall quota
174+
exceeders.erase(EntireDatabaseTag);
175+
} else {
176+
// ignore the statuses of the separate storage pool quotas
177+
TMap<TString, EDiskUsageStatus> onlyOverallStatus;
178+
if (auto* overallStatus = exceeders.FindPtr(EntireDatabaseTag)) {
179+
onlyOverallStatus.emplace(EntireDatabaseTag, *overallStatus);
180+
}
181+
std::swap(exceeders, onlyOverallStatus);
182+
}
171183
UNIT_ASSERT_VALUES_EQUAL_C(exceeders, expectedExceeders,
172184
debugHint << ", subdomain's disk space usage:\n" << desc.GetDiskSpaceUsage().DebugString()
173185
);
@@ -181,9 +193,9 @@ void CheckQuotaExceedance(TTestActorRuntime& runtime,
181193
bool expectExceeded,
182194
const TString& debugHint = ""
183195
) {
184-
TestDescribeResult(DescribePath(runtime, schemeShard, pathToSubdomain),
185-
{ LsCheckDiskQuotaExceeded(expectExceeded, debugHint) }
186-
);
196+
TestDescribeResult(DescribePath(runtime, schemeShard, pathToSubdomain), {
197+
LsCheckDiskQuotaExceeded(expectExceeded, debugHint)
198+
});
187199
}
188200

189201
void CheckQuotaExceedance(TTestActorRuntime& runtime,
@@ -192,9 +204,13 @@ void CheckQuotaExceedance(TTestActorRuntime& runtime,
192204
const TMap<TString, EDiskUsageStatus>& expectedExceeders,
193205
const TString& debugHint = ""
194206
) {
195-
TestDescribeResult(DescribePath(runtime, schemeShard, pathToSubdomain),
196-
{ LsCheckDiskQuotaExceeded(expectedExceeders, debugHint) }
197-
);
207+
TestDescribeResult(DescribePath(runtime, schemeShard, pathToSubdomain), {
208+
LsCheckDiskQuotaExceeded(
209+
expectedExceeders,
210+
runtime.GetAppData().FeatureFlags.GetEnableSeparateDiskSpaceQuotas(),
211+
debugHint
212+
)
213+
});
198214
}
199215

200216
TVector<ui64> GetTableShards(TTestActorRuntime& runtime,
@@ -3207,6 +3223,8 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) {
32073223
opts.EnableTopicDiskSubDomainQuota(false);
32083224

32093225
TTestEnv env(runtime, opts);
3226+
runtime.GetAppData().FeatureFlags.SetEnableSeparateDiskSpaceQuotas(false);
3227+
32103228
ui64 txId = 100;
32113229

32123230
auto waitForTableStats = [&](ui32 shards) {
@@ -3435,6 +3453,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) {
34353453
runtime.SetLogPriority(NKikimrServices::PERSQUEUE_READ_BALANCER, NLog::PRI_TRACE);
34363454

34373455
runtime.GetAppData().PQConfig.SetBalancerWakeupIntervalSec(1);
3456+
runtime.GetAppData().FeatureFlags.SetEnableSeparateDiskSpaceQuotas(false);
34383457

34393458
ui64 txId = 100;
34403459

@@ -3482,7 +3501,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) {
34823501

34833502
auto stats = NPQ::GetReadBalancerPeriodicTopicStats(runtime, balancerId);
34843503
UNIT_ASSERT_EQUAL_C(false, stats->Record.GetSubDomainOutOfSpace(), "SubDomainOutOfSpace from ReadBalancer");
3485-
3504+
34863505
auto msg = TString(24_MB, '_');
34873506

34883507
ui32 seqNo = 100;
@@ -3517,7 +3536,9 @@ Y_UNIT_TEST_SUITE(TStoragePoolsQuotasTest) {
35173536
opts.EnablePersistentPartitionStats(true);
35183537
opts.EnableBackgroundCompaction(false);
35193538
TTestEnv env(runtime, opts);
3520-
3539+
3540+
runtime.GetAppData().FeatureFlags.SetEnableSeparateDiskSpaceQuotas(true);
3541+
35213542
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
35223543
NDataShard::gDbStatsDataSizeResolution = 1;
35233544
NDataShard::gDbStatsRowCountResolution = 1;
@@ -3610,7 +3631,7 @@ Y_UNIT_TEST_SUITE(TStoragePoolsQuotasTest) {
36103631
UpdateRow(runtime, "SomeTable", 1, "some_value_for_the_key", shards[0]);
36113632
{
36123633
const auto tableStats = WaitTableStats(runtime, shards[0]).GetTableStats();
3613-
// channels' usage statistics appears only after a table compaction
3634+
// channels' usage statistics appears only after a table compaction
36143635
UNIT_ASSERT_VALUES_EQUAL_C(tableStats.ChannelsSize(), 0, tableStats.DebugString());
36153636
}
36163637
CheckQuotaExceedance(runtime, tenantSchemeShard, "/MyRoot/SomeDatabase", false, DEBUG_HINT);
@@ -3636,7 +3657,7 @@ Y_UNIT_TEST_SUITE(TStoragePoolsQuotasTest) {
36363657

36373658
TTestEnvOptions opts;
36383659
TTestEnv env(runtime, opts);
3639-
3660+
36403661
ui64 txId = 100;
36413662

36423663
constexpr const char* databaseDescription = R"(
@@ -3678,8 +3699,8 @@ Y_UNIT_TEST_SUITE(TStoragePoolsQuotasTest) {
36783699

36793700
// This test might start failing, because disk space usage of the created table might change
36803701
// due to changes in the storage implementation.
3681-
// To fix the test you need to update canonical quotas and / or batch sizes.
3682-
Y_UNIT_TEST_FLAG(DifferentQuotasInteraction, IsExternalSubdomain) {
3702+
// To fix the test you need to update canonical quotas and the content of the table.
3703+
Y_UNIT_TEST_FLAGS(DifferentQuotasInteraction, IsExternalSubdomain, EnableSeparateQuotas) {
36833704
TTestBasicRuntime runtime;
36843705
runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NLog::PRI_TRACE);
36853706

@@ -3688,8 +3709,7 @@ Y_UNIT_TEST_SUITE(TStoragePoolsQuotasTest) {
36883709
opts.EnablePersistentPartitionStats(true);
36893710
opts.EnableBackgroundCompaction(false);
36903711
TTestEnv env(runtime, opts);
3691-
bool bTreeIndex = runtime.GetAppData().FeatureFlags.GetEnableLocalDBBtreeIndex();
3692-
3712+
36933713
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
36943714
NDataShard::gDbStatsDataSizeResolution = 1;
36953715
NDataShard::gDbStatsRowCountResolution = 1;
@@ -3798,43 +3818,46 @@ Y_UNIT_TEST_SUITE(TStoragePoolsQuotasTest) {
37983818

37993819
const auto updateAndCheck = [&](ui32 rowsToUpdate,
38003820
const TString& value,
3801-
bool compact,
38023821
const TMap<TString, EDiskUsageStatus>& expectedExceeders,
38033822
const TString& debugHint = ""
38043823
) {
38053824
for (ui32 i = 0; i < rowsToUpdate; ++i) {
38063825
UpdateRow(runtime, "SomeTable", i, value, shards[0]);
38073826
}
3808-
if (compact) {
3809-
CompactTableAndCheckResult(runtime, shards[0], tableId);
3810-
}
3827+
CompactTableAndCheckResult(runtime, shards[0], tableId);
38113828
WaitTableStats(runtime, shards[0]);
38123829
CheckQuotaExceedance(runtime, tenantSchemeShard, "/MyRoot/SomeDatabase", expectedExceeders, debugHint);
38133830
};
38143831

3815-
// Warning: calculated empirically, might need an update if the test fails.
3816-
// The logic of the test expects:
3817-
// batchSizes[0] <= batchSizes[1] <= batchSizes[2],
3818-
// because rows are never deleted, only updated.
3819-
const std::array<ui32, 3> batchSizes = {25, 35, bTreeIndex ? 60u : 50u};
3820-
3821-
constexpr const char* longText = "this_text_is_very_long_and_takes_a_lot_of_disk_space";
3822-
constexpr const char* middleLengthText = "this_text_is_significantly_shorter";
3823-
3824-
// Test scenario:
3825-
// 1) break only the entire database hard quota, don't break others,
3826-
updateAndCheck(batchSizes[0], longText, false, {{EntireDatabaseTag, EDiskUsageStatus::AboveHardQuota}}, DEBUG_HINT);
3827-
updateAndCheck(0, "", true, {}, DEBUG_HINT);
3828-
3829-
// 2) break only the large_kind hard quota, don't break other hard quotas,
3830-
updateAndCheck(batchSizes[1], longText, true,
3831-
{{"large_kind", EDiskUsageStatus::AboveHardQuota}, {EntireDatabaseTag, EDiskUsageStatus::InBetween}}, DEBUG_HINT
3832-
);
3833-
updateAndCheck(batchSizes[1], middleLengthText, true, {{"large_kind", EDiskUsageStatus::InBetween}}, DEBUG_HINT);
3834-
updateAndCheck(batchSizes[1], "extra_short_text", true, {}, DEBUG_HINT);
3832+
// Warning: calculated empirically, might need an update if the test fails!
3833+
constexpr ui32 lessRows = 37u;
3834+
const ui32 moreRows = runtime.GetAppData().FeatureFlags.GetEnableLocalDBBtreeIndex() ? 60u : 50u;
3835+
3836+
const TString longText = TString(64, 'a');;
3837+
const TString mediumText = TString(32, 'a');
3838+
const TString shortText = TString(16, 'a');
3839+
const TString tinyText = TString(8, 'a');
38353840

3836-
// 3) break only the fast_kind hard quota, don't break others.
3837-
updateAndCheck(batchSizes[2], "shortest", true, {{"fast_kind", EDiskUsageStatus::AboveHardQuota}}, DEBUG_HINT);
3841+
runtime.GetAppData().FeatureFlags.SetEnableSeparateDiskSpaceQuotas(EnableSeparateQuotas);
3842+
if (!EnableSeparateQuotas) {
3843+
// write a lot of data to break the overall hard quota
3844+
updateAndCheck(lessRows, longText, {{EntireDatabaseTag, EDiskUsageStatus::AboveHardQuota}}, DEBUG_HINT);
3845+
} else {
3846+
// There are two columns in the table: key and value. Key is stored at the fast storage, value at the large storage.
3847+
// We can:
3848+
// - simultaneously increase the consumption of the both storage pools by increasing the number of rows in the table
3849+
// - increase or decrease the consumption of the large storage by making the value longer / shorter
3850+
// Test scenario:
3851+
// 1) write a small number of rows (little fast storage consumption), but a long text that breaks the large kind hard quota
3852+
// 2) the same small number of rows, but a medium text that gets the large kind storage consumption in between the soft and the large quotas
3853+
// 3) the same small number of rows, but a short text that gets the large kind storage consumption below the soft quota
3854+
// 4) a bigger number of rows, but a tiny text to break only the fast kind hard quota
3855+
updateAndCheck(lessRows, longText, {{"large_kind", EDiskUsageStatus::AboveHardQuota}}, DEBUG_HINT);
3856+
updateAndCheck(lessRows, mediumText, {{"large_kind", EDiskUsageStatus::InBetween}}, DEBUG_HINT);
3857+
updateAndCheck(lessRows, shortText, {}, DEBUG_HINT);
3858+
3859+
updateAndCheck(moreRows, tinyText, {{"fast_kind", EDiskUsageStatus::AboveHardQuota}}, DEBUG_HINT);
3860+
}
38383861

38393862
// step 4: drop the table
38403863
TestDropTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/SomeDatabase", "SomeTable");

‎ydb/services/ydb/ydb_ut.cpp

Lines changed: 37 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5667,32 +5667,51 @@ Y_UNIT_TEST(DisableWritesToDatabase) {
56675667
), false
56685668
);
56695669

5670-
ExecSQL(server, sender, Sprintf(R"(
5671-
UPSERT INTO `%s` (Key, Value) VALUES (1u, "Foo");
5672-
)", table.c_str()
5673-
)
5674-
);
5670+
auto upsert = [&](
5671+
const TString& table, const TString& row,
5672+
Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS
5673+
) {
5674+
ExecSQL(server, sender, Sprintf(R"(
5675+
UPSERT INTO `%s` (Key, Value) VALUES (%s);
5676+
)", table.c_str(), row.c_str()
5677+
), true, expectedStatus
5678+
);
5679+
};
5680+
5681+
upsert(table, "1u, \"Foo\"");
56755682

56765683
auto shards = GetTableShards(server, sender, table);
56775684
UNIT_ASSERT_VALUES_EQUAL(shards.size(), 1);
56785685
auto& datashard = shards[0];
56795686
auto tableId = ResolveTableId(server, sender, table);
5680-
56815687
// Compaction is a must. Table stats are missing channels usage statistics until the table is compacted at least once.
56825688
CompactTableAndCheckResult(runtime, datashard, tableId);
5683-
WaitTableStats(runtime, datashard, 1);
56845689

5685-
ExecSQL(server, sender, Sprintf(R"(
5686-
UPSERT INTO `%s` (Key, Value) VALUES (2u, "Bar");
5687-
)", table.c_str()
5688-
), true, Ydb::StatusIds::UNAVAILABLE
5689-
);
5690-
auto schemeEntry = Navigate(runtime, sender, tenantPath, NSchemeCache::TSchemeCacheNavigate::EOp::OpPath)->ResultSet.at(0);
5691-
UNIT_ASSERT_C(schemeEntry.DomainDescription, schemeEntry.ToString());
5692-
auto& domainDescription = schemeEntry.DomainDescription->Description;
5693-
UNIT_ASSERT_C(domainDescription.HasDomainState(), domainDescription.DebugString());
5694-
bool quotaExceeded = domainDescription.GetDomainState().GetDiskQuotaExceeded();
5695-
UNIT_ASSERT_C(quotaExceeded, domainDescription.DebugString());
5690+
auto checkDatabaseState = [&](const TString& database, bool expectedQuotaExceeded) {
5691+
auto schemeEntry = Navigate(
5692+
runtime, sender, database, NSchemeCache::TSchemeCacheNavigate::EOp::OpPath
5693+
)->ResultSet.at(0);
5694+
UNIT_ASSERT_C(schemeEntry.DomainDescription, schemeEntry.ToString());
5695+
auto& domainDescription = schemeEntry.DomainDescription->Description;
5696+
bool quotaExceeded = domainDescription.GetDomainState().GetDiskQuotaExceeded();
5697+
UNIT_ASSERT_VALUES_EQUAL_C(quotaExceeded, expectedQuotaExceeded, domainDescription.DebugString());
5698+
};
5699+
5700+
// try upsert when the feature flag is enabled
5701+
{
5702+
runtime.GetAppData().FeatureFlags.SetEnableSeparateDiskSpaceQuotas(true);
5703+
WaitTableStats(runtime, datashard, 1);
5704+
upsert(table, "2u, \"Bar\"", Ydb::StatusIds::UNAVAILABLE);
5705+
checkDatabaseState(tenantPath, true);
5706+
}
5707+
5708+
// try upsert when the feature flag is disabled
5709+
{
5710+
runtime.GetAppData().FeatureFlags.SetEnableSeparateDiskSpaceQuotas(false);
5711+
WaitTableStats(runtime, datashard, 1);
5712+
upsert(table, "2u, \"Bar\"");
5713+
checkDatabaseState(tenantPath, false);
5714+
}
56965715
}
56975716

56985717
}

0 commit comments

Comments
 (0)