Skip to content

Commit dd776b0

Browse files
va-kuznecovVPolka
va-kuznecov
and
VPolka
authored
Add ICB to control VDisk compaction InFlight (#20508)
Co-authored-by: Polina Volosnikova <vpolka@nebius.com> Co-authored-by: VPolka <pvolosnikova98@gmail.com>
1 parent 2a57035 commit dd776b0

File tree

12 files changed

+81
-50
lines changed

12 files changed

+81
-50
lines changed

‎ydb/core/blobstorage/nodewarden/node_warden_impl.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ TNodeWarden::TNodeWarden(const TIntrusivePtr<TNodeWardenConfig> &cfg)
3535
, DefaultHugeGarbagePerMille(300, 1, 1000)
3636
, HugeDefragFreeSpaceBorderPerMille(260, 1, 1000)
3737
, MaxChunksToDefragInflight(10, 1, 50)
38+
, FreshCompMaxInFlightWrites(10, 1, 1000)
39+
, FreshCompMaxInFlightReads(10, 1, 1000)
40+
, HullCompMaxInFlightWrites(10, 1, 1000)
41+
, HullCompMaxInFlightReads(20, 1, 1000)
3842
, ThrottlingDryRun(1, 0, 1)
3943
, ThrottlingMinLevel0SstCount(100, 1, 100000)
4044
, ThrottlingMaxLevel0SstCount(250, 1, 100000)
@@ -370,6 +374,10 @@ void TNodeWarden::Bootstrap() {
370374
icb->RegisterSharedControl(DefaultHugeGarbagePerMille, "VDiskControls.DefaultHugeGarbagePerMille");
371375
icb->RegisterSharedControl(HugeDefragFreeSpaceBorderPerMille, "VDiskControls.HugeDefragFreeSpaceBorderPerMille");
372376
icb->RegisterSharedControl(MaxChunksToDefragInflight, "VDiskControls.MaxChunksToDefragInflight");
377+
icb->RegisterSharedControl(FreshCompMaxInFlightWrites, "VDiskControls.FreshCompMaxInFlightWrites");
378+
icb->RegisterSharedControl(FreshCompMaxInFlightReads, "VDiskControls.FreshCompMaxInFlightReads");
379+
icb->RegisterSharedControl(HullCompMaxInFlightWrites, "VDiskControls.HullCompMaxInFlightWrites");
380+
icb->RegisterSharedControl(HullCompMaxInFlightReads, "VDiskControls.HullCompMaxInFlightReads");
373381

374382
icb->RegisterSharedControl(ThrottlingDryRun, "VDiskControls.ThrottlingDryRun");
375383
icb->RegisterSharedControl(ThrottlingMinLevel0SstCount, "VDiskControls.ThrottlingMinLevel0SstCount");

‎ydb/core/blobstorage/nodewarden/node_warden_impl.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,10 @@ namespace NKikimr::NStorage {
206206
TControlWrapper DefaultHugeGarbagePerMille;
207207
TControlWrapper HugeDefragFreeSpaceBorderPerMille;
208208
TControlWrapper MaxChunksToDefragInflight;
209+
TControlWrapper FreshCompMaxInFlightWrites;
210+
TControlWrapper FreshCompMaxInFlightReads;
211+
TControlWrapper HullCompMaxInFlightWrites;
212+
TControlWrapper HullCompMaxInFlightReads;
209213

210214
TControlWrapper ThrottlingDryRun;
211215
TControlWrapper ThrottlingMinLevel0SstCount;

‎ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,10 @@ namespace NKikimr::NStorage {
196196
vdiskConfig->DefaultHugeGarbagePerMille = DefaultHugeGarbagePerMille;
197197
vdiskConfig->HugeDefragFreeSpaceBorderPerMille = HugeDefragFreeSpaceBorderPerMille;
198198
vdiskConfig->MaxChunksToDefragInflight = MaxChunksToDefragInflight;
199+
vdiskConfig->FreshCompMaxInFlightWrites = FreshCompMaxInFlightWrites;
200+
vdiskConfig->FreshCompMaxInFlightReads = FreshCompMaxInFlightReads;
201+
vdiskConfig->HullCompMaxInFlightWrites = HullCompMaxInFlightWrites;
202+
vdiskConfig->HullCompMaxInFlightReads = HullCompMaxInFlightReads;
199203

200204
vdiskConfig->EnableLocalSyncLogDataCutting = EnableLocalSyncLogDataCutting;
201205
if (deviceType == NPDisk::EDeviceType::DEVICE_TYPE_ROT) {

‎ydb/core/blobstorage/vdisk/common/vdisk_config.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,11 @@ namespace NKikimr {
8484
HandoffTimeout = TDuration::Seconds(10);
8585
RunRepl = !baseInfo.ReadOnly;
8686

87-
ReplMaxTimeToMakeProgress = VDiskPerformance.at(baseInfo.DeviceType).ReplMaxTimeToMakeProgress;
87+
if (const auto& perf = VDiskPerformance.find(baseInfo.DeviceType); perf != VDiskPerformance.end()) {
88+
ReplMaxTimeToMakeProgress = perf->second.ReplMaxTimeToMakeProgress;
89+
} else {
90+
ReplMaxTimeToMakeProgress = TDuration::Minutes(180);
91+
}
8892

8993
SkeletonFrontGets_MaxInFlightCount = 24;
9094
SkeletonFrontGets_MaxInFlightCost = 200000000; // 200ms

‎ydb/core/blobstorage/vdisk/common/vdisk_config.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,10 +126,10 @@ namespace NKikimr {
126126
ui32 HullCompSortedPartsNum;
127127
double HullCompLevelRateThreshold;
128128
double HullCompFreeSpaceThreshold;
129-
ui32 FreshCompMaxInFlightWrites;
130-
ui32 FreshCompMaxInFlightReads;
131-
ui32 HullCompMaxInFlightWrites;
132-
ui32 HullCompMaxInFlightReads;
129+
TControlWrapper FreshCompMaxInFlightWrites;
130+
TControlWrapper FreshCompMaxInFlightReads;
131+
TControlWrapper HullCompMaxInFlightWrites;
132+
TControlWrapper HullCompMaxInFlightReads;
133133
double HullCompReadBatchEfficiencyThreshold;
134134
ui64 AnubisOsirisMaxInFly;
135135
bool AddHeader;

‎ydb/core/blobstorage/vdisk/hulldb/base/blobstorage_hulldefs.cpp

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -85,13 +85,12 @@ namespace NKikimr {
8585
str << "{Id# " << Id << "}";
8686
}
8787

88-
THullCtx::THullCtx(TVDiskContextPtr vctx, ui32 chunkSize, ui32 compWorthReadSize, bool freshCompaction,
89-
bool gcOnlySynced, bool allowKeepFlags, bool barrierValidation, ui32 hullSstSizeInChunksFresh,
90-
ui32 hullSstSizeInChunksLevel, double hullCompFreeSpaceThreshold, ui32 freshCompMaxInFlightWrites,
91-
ui32 freshCompMaxInFlightReads, ui32 hullCompMaxInFlightWrites, ui32 hullCompMaxInFlightReads,
92-
double hullCompReadBatchEfficiencyThreshold, TDuration hullCompStorageRatioCalcPeriod,
93-
TDuration hullCompStorageRatioMaxCalcDuration, bool addHeader)
88+
THullCtx::THullCtx(TVDiskContextPtr vctx, const TIntrusivePtr<TVDiskConfig> vcfg, ui32 chunkSize, ui32 compWorthReadSize,
89+
bool freshCompaction, bool gcOnlySynced, bool allowKeepFlags, bool barrierValidation, ui32 hullSstSizeInChunksFresh,
90+
ui32 hullSstSizeInChunksLevel, double hullCompFreeSpaceThreshold, double hullCompReadBatchEfficiencyThreshold,
91+
TDuration hullCompStorageRatioCalcPeriod, TDuration hullCompStorageRatioMaxCalcDuration, bool addHeader)
9492
: VCtx(std::move(vctx))
93+
, VCfg(vcfg)
9594
, IngressCache(TIngressCache::Create(VCtx->Top, VCtx->ShortSelfVDisk))
9695
, ChunkSize(chunkSize)
9796
, CompWorthReadSize(compWorthReadSize)
@@ -102,10 +101,6 @@ namespace NKikimr {
102101
, HullSstSizeInChunksFresh(hullSstSizeInChunksFresh)
103102
, HullSstSizeInChunksLevel(hullSstSizeInChunksLevel)
104103
, HullCompFreeSpaceThreshold(hullCompFreeSpaceThreshold)
105-
, FreshCompMaxInFlightWrites(freshCompMaxInFlightWrites)
106-
, FreshCompMaxInFlightReads(freshCompMaxInFlightReads)
107-
, HullCompMaxInFlightWrites(hullCompMaxInFlightWrites)
108-
, HullCompMaxInFlightReads(hullCompMaxInFlightReads)
109104
, HullCompReadBatchEfficiencyThreshold(hullCompReadBatchEfficiencyThreshold)
110105
, HullCompStorageRatioCalcPeriod(hullCompStorageRatioCalcPeriod)
111106
, HullCompStorageRatioMaxCalcDuration(hullCompStorageRatioMaxCalcDuration)

‎ydb/core/blobstorage/vdisk/hulldb/base/blobstorage_hulldefs.h

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ namespace NKikimr {
121121
///////////////////////////////////////////////////////////////////////////////////////
122122
struct THullCtx : public TThrRefBase {
123123
TVDiskContextPtr VCtx;
124+
const TIntrusivePtr<TVDiskConfig> VCfg;
124125
const TIntrusivePtr<TIngressCache> IngressCache;
125126
const ui32 ChunkSize;
126127
const ui32 CompWorthReadSize;
@@ -131,10 +132,6 @@ namespace NKikimr {
131132
const ui32 HullSstSizeInChunksFresh;
132133
const ui32 HullSstSizeInChunksLevel;
133134
const double HullCompFreeSpaceThreshold;
134-
const ui32 FreshCompMaxInFlightWrites;
135-
const ui32 FreshCompMaxInFlightReads;
136-
const ui32 HullCompMaxInFlightWrites;
137-
const ui32 HullCompMaxInFlightReads;
138135
const double HullCompReadBatchEfficiencyThreshold;
139136
const TDuration HullCompStorageRatioCalcPeriod;
140137
const TDuration HullCompStorageRatioMaxCalcDuration;
@@ -146,6 +143,7 @@ namespace NKikimr {
146143

147144
THullCtx(
148145
TVDiskContextPtr vctx,
146+
const TIntrusivePtr<TVDiskConfig> vcfg,
149147
ui32 chunkSize,
150148
ui32 compWorthReadSize,
151149
bool freshCompaction,
@@ -155,10 +153,6 @@ namespace NKikimr {
155153
ui32 hullSstSizeInChunksFresh,
156154
ui32 hullSstSizeInChunksLevel,
157155
double hullCompFreeSpaceThreshold,
158-
ui32 freshCompMaxInFlightWrites,
159-
ui32 freshCompMaxInFlightReads,
160-
ui32 hullCompMaxInFlightWrites,
161-
ui32 hullCompMaxInFlightReads,
162156
double hullCompReadBatchEfficiencyThreshold,
163157
TDuration hullCompStorageRatioCalcPeriod,
164158
TDuration hullCompStorageRatioMaxCalcDuration,

‎ydb/core/blobstorage/vdisk/hulldb/base/hullds_ut.h

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,13 @@ namespace NKikimr {
1212
: ChunkSize(chunkSize)
1313
, CompWorthReadSize(compWorthReadSize)
1414
, GroupInfo(TBlobStorageGroupType::ErasureMirror3, 2, 4)
15+
, VCfg(TVDiskConfig::TBaseInfo::SampleForTests())
1516
, VCtx(new TVDiskContext(TActorId(), GroupInfo.PickTopology(), new ::NMonitoring::TDynamicCounters(),
1617
TVDiskID(), nullptr, NPDisk::DEVICE_TYPE_UNKNOWN))
1718
, HullCtx(
1819
new THullCtx(
1920
VCtx,
21+
MakeIntrusive<TVDiskConfig>(VCfg),
2022
ChunkSize,
2123
CompWorthReadSize,
2224
true,
@@ -26,14 +28,10 @@ namespace NKikimr {
2628
1, // HullSstSizeInChunksFresh
2729
1, // HullSstSizeInChunksLevel
2830
2.0,
29-
10, // FreshCompMaxInFlightWrites
30-
10, // FreshCompMaxInFlightReads
31-
10, // HullCompMaxInFlightWrites
32-
20, // HullCompMaxInFlightReads
3331
0.5,
3432
TDuration::Minutes(5),
3533
TDuration::Seconds(1),
36-
true)) // AddHeader
34+
true)) // AddHeader
3735
, LevelIndexSettings(
3836
HullCtx,
3937
8u, // Level0MaxSstsAtOnce
@@ -61,6 +59,7 @@ namespace NKikimr {
6159
const ui32 ChunkSize;
6260
const ui64 CompWorthReadSize;
6361
TBlobStorageGroupInfo GroupInfo;
62+
TVDiskConfig VCfg;
6463
TVDiskContextPtr VCtx;
6564
THullCtxPtr HullCtx;
6665
TLevelIndexSettings LevelIndexSettings;

‎ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompactworker.h

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -164,15 +164,9 @@ namespace NKikimr {
164164
// number of currently unresponded write requests
165165
ui32 InFlightWrites = 0;
166166

167-
// maximum number of such requests
168-
ui32 MaxInFlightWrites;
169-
170167
// number of currently unresponded read requests
171168
ui32 InFlightReads = 0;
172169

173-
// maximum number of such requests
174-
ui32 MaxInFlightReads = 0;
175-
176170
// vector of freed huge blobs
177171
TDiskPartVec FreedHugeBlobs;
178172
TDiskPartVec AllocatedHugeBlobs;
@@ -324,14 +318,10 @@ namespace NKikimr {
324318
{
325319
if (IsFresh) {
326320
ChunksToUse = HullCtx->HullSstSizeInChunksFresh;
327-
MaxInFlightWrites = HullCtx->FreshCompMaxInFlightWrites;
328-
MaxInFlightReads = HullCtx->FreshCompMaxInFlightReads;
329321
ReadsInFlight = &LevelIndex->FreshCompReadsInFlight;
330322
WritesInFlight = &LevelIndex->FreshCompWritesInFlight;
331323
} else {
332324
ChunksToUse = HullCtx->HullSstSizeInChunksLevel;
333-
MaxInFlightWrites = HullCtx->HullCompMaxInFlightWrites;
334-
MaxInFlightReads = HullCtx->HullCompMaxInFlightReads;
335325
ReadsInFlight = &LevelIndex->HullCompReadsInFlight;
336326
WritesInFlight = &LevelIndex->HullCompWritesInFlight;
337327
}
@@ -435,11 +425,12 @@ namespace NKikimr {
435425
Y_VERIFY_S(!WriterPtr->GetPendingMessage(), HullCtx->VCtx->VDiskLogPrefix);
436426
WriterPtr.reset();
437427
} else {
438-
Y_VERIFY_S(InFlightWrites == MaxInFlightWrites, HullCtx->VCtx->VDiskLogPrefix);
428+
Y_VERIFY_S(InFlightWrites == GetMaxInFlightWrites(), HullCtx->VCtx->VDiskLogPrefix);
439429
return false;
440430
}
441431
break;
442432
}
433+
443434

444435
case EState::WaitForPendingRequests:
445436
// wait until all writes succeed
@@ -712,7 +703,7 @@ namespace NKikimr {
712703
bool FlushSST() {
713704
// try to flush some more data; if the flush fails, it means that we have reached in flight write limit and
714705
// there is nothing to do here now, so we return
715-
if (!WriterPtr->FlushNext(FirstLsn, LastLsn, MaxInFlightWrites - InFlightWrites)) {
706+
if (!WriterPtr->FlushNext(FirstLsn, LastLsn, GetMaxInFlightWrites() - InFlightWrites)) {
716707
return false;
717708
}
718709

@@ -727,12 +718,12 @@ namespace NKikimr {
727718
void ProcessPendingMessages(TVector<std::unique_ptr<IEventBase>>& msgsForYard) {
728719
// ensure that we have writer
729720
Y_VERIFY_S(WriterPtr, HullCtx->VCtx->VDiskLogPrefix);
730-
Y_VERIFY_S(MaxInFlightWrites, HullCtx->VCtx->VDiskLogPrefix);
731-
Y_VERIFY_S(MaxInFlightReads, HullCtx->VCtx->VDiskLogPrefix);
721+
Y_VERIFY_S(GetMaxInFlightWrites(), HullCtx->VCtx->VDiskLogPrefix);
722+
Y_VERIFY_S(GetMaxInFlightReads(), HullCtx->VCtx->VDiskLogPrefix);
732723

733724
// send new messages until we reach in flight limit
734725
std::unique_ptr<NPDisk::TEvChunkWrite> msg;
735-
while (InFlightWrites < MaxInFlightWrites && (msg = GetPendingWriteMessage())) {
726+
while (InFlightWrites < GetMaxInFlightWrites() && (msg = GetPendingWriteMessage())) {
736727
HullCtx->VCtx->CountCompactionCost(*msg);
737728
Statistics.Update(msg.get());
738729
msgsForYard.push_back(std::move(msg));
@@ -741,7 +732,7 @@ namespace NKikimr {
741732
}
742733

743734
std::unique_ptr<NPDisk::TEvChunkRead> readMsg;
744-
while (InFlightReads < MaxInFlightReads && (readMsg = ReadBatcher.GetPendingMessage(
735+
while (InFlightReads < GetMaxInFlightReads() && (readMsg = ReadBatcher.GetPendingMessage(
745736
PDiskCtx->Dsk->Owner, PDiskCtx->Dsk->OwnerRound, NPriRead::HullComp))) {
746737
HullCtx->VCtx->CountCompactionCost(*readMsg);
747738
Statistics.Update(readMsg.get());
@@ -768,6 +759,14 @@ namespace NKikimr {
768759
ChunkReservePending += num;
769760
return std::make_unique<NPDisk::TEvChunkReserve>(PDiskCtx->Dsk->Owner, PDiskCtx->Dsk->OwnerRound, num);
770761
}
762+
763+
ui32 GetMaxInFlightWrites() {
764+
return IsFresh ? HullCtx->VCfg->FreshCompMaxInFlightWrites : HullCtx->VCfg->HullCompMaxInFlightWrites;
765+
}
766+
767+
ui32 GetMaxInFlightReads() {
768+
return IsFresh ? (ui32) HullCtx->VCfg->FreshCompMaxInFlightReads : (ui32) HullCtx->VCfg->HullCompMaxInFlightReads;
769+
}
771770
};
772771

773772
} // NKikimr

‎ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -506,6 +506,7 @@ namespace NKikimr {
506506
Y_VERIFY_S(LocRecCtx->VCtx && LocRecCtx->VCtx->Top, LocRecCtx->VCtx->VDiskLogPrefix);
507507
auto hullCtx = MakeIntrusive<THullCtx>(
508508
LocRecCtx->VCtx,
509+
Config,
509510
ui32(LocRecCtx->PDiskCtx->Dsk->ChunkSize),
510511
ui32(LocRecCtx->PDiskCtx->Dsk->PrefetchSizeBytes),
511512
Config->FreshCompaction && !Config->BaseInfo.ReadOnly,
@@ -515,10 +516,6 @@ namespace NKikimr {
515516
Config->HullSstSizeInChunksFresh,
516517
Config->HullSstSizeInChunksLevel,
517518
Config->HullCompFreeSpaceThreshold,
518-
Config->FreshCompMaxInFlightWrites,
519-
Config->FreshCompMaxInFlightReads,
520-
Config->HullCompMaxInFlightWrites,
521-
Config->HullCompMaxInFlightReads,
522519
Config->HullCompReadBatchEfficiencyThreshold,
523520
Config->HullCompStorageRatioCalcPeriod,
524521
Config->HullCompStorageRatioMaxCalcDuration,

‎ydb/core/blobstorage/vdisk/repl/blobstorage_hullreplwritesst_ut.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,9 @@ TVDiskContextPtr CreateVDiskContext(const TBlobStorageGroupInfo& info) {
3838
}
3939

4040
TIntrusivePtr<THullCtx> CreateHullCtx(const TBlobStorageGroupInfo& info, ui32 chunkSize, ui32 compWorthReadSize) {
41-
return MakeIntrusive<THullCtx>(CreateVDiskContext(info), chunkSize, compWorthReadSize, true, true, true, true, 1u,
42-
1u, 2.0, 10u, 10u, 10u, 20u, 0.5, TDuration::Minutes(5), TDuration::Seconds(1), true);
41+
auto baseInfo = TVDiskConfig::TBaseInfo::SampleForTests();
42+
return MakeIntrusive<THullCtx>(CreateVDiskContext(info), MakeIntrusive<TVDiskConfig>(baseInfo), chunkSize, compWorthReadSize, true, true, true, true, 1u,
43+
1u, 2.0, 0.5, TDuration::Minutes(5), TDuration::Seconds(1), true);
4344
}
4445

4546
TIntrusivePtr<THullDs> CreateHullDs(const TBlobStorageGroupInfo& info) {

‎ydb/core/protos/config.proto

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1549,6 +1549,32 @@ message TImmediateControlsConfig {
15491549
MinValue: 0,
15501550
MaxValue: 1000,
15511551
DefaultValue: 0 }];
1552+
1553+
reserved 28;
1554+
1555+
optional uint64 FreshCompMaxInFlightWrites = 29 [(ControlOptions) = {
1556+
Description: "Max writes inflight for fresh level compaction",
1557+
MinValue: 1,
1558+
MaxValue: 1000,
1559+
DefaultValue: 10 }];
1560+
1561+
optional uint64 FreshCompMaxInFlightReads = 30 [(ControlOptions) = {
1562+
Description: "Max reads inflight for fresh level compaction",
1563+
MinValue: 1,
1564+
MaxValue: 1000,
1565+
DefaultValue: 10 }];
1566+
1567+
optional uint64 HullCompMaxInFlightWrites = 31 [(ControlOptions) = {
1568+
Description: "Max writes inflight for level compaction",
1569+
MinValue: 1,
1570+
MaxValue: 1000,
1571+
DefaultValue: 10 }];
1572+
1573+
optional uint64 HullCompMaxInFlightReads = 32 [(ControlOptions) = {
1574+
Description: "Max reads inflight for level compaction",
1575+
MinValue: 1,
1576+
MaxValue: 1000,
1577+
DefaultValue: 20 }];
15521578
}
15531579

15541580
message TTabletControls {

0 commit comments

Comments
 (0)