Skip to content

Commit 33d586d

Browse files
alexd65536alexd65536
authored
add throttling by inplaced blobs total size; add counters (#12829)
1 parent ce4b678 commit 33d586d

File tree

4 files changed

+109
-32
lines changed

4 files changed

+109
-32
lines changed

‎ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,11 @@ public:
135135

136136
COUNTER_INIT_IF_EXTENDED(FreshSatisfactionRankPercent, false);
137137
COUNTER_INIT_IF_EXTENDED(LevelSatisfactionRankPercent, false);
138+
139+
COUNTER_INIT_IF_EXTENDED(ThrottlingCurrentSpeedLimit, false);
140+
COUNTER_INIT_IF_EXTENDED(ThrottlingIsActive, false);
141+
COUNTER_INIT_IF_EXTENDED(ThrottlingLevel0SstCount, false);
142+
COUNTER_INIT_IF_EXTENDED(ThrottlingAllLevelsInplacedSize, false);
138143
}
139144

140145
COUNTER_DEF(EmergencyMovedPatchQueueItems);
@@ -153,6 +158,11 @@ public:
153158

154159
COUNTER_DEF(FreshSatisfactionRankPercent);
155160
COUNTER_DEF(LevelSatisfactionRankPercent);
161+
162+
COUNTER_DEF(ThrottlingCurrentSpeedLimit);
163+
COUNTER_DEF(ThrottlingIsActive);
164+
COUNTER_DEF(ThrottlingLevel0SstCount);
165+
COUNTER_DEF(ThrottlingAllLevelsInplacedSize);
156166
};
157167

158168
///////////////////////////////////////////////////////////////////////////////////
@@ -455,6 +465,7 @@ public:
455465
TLsmLevelGroup Level9to16;
456466
TLsmLevelGroup Level17;
457467
TLsmLevelGroup Level18;
468+
TLsmLevelGroup Level19;
458469

459470
TLsmAllLevelsStat(const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters)
460471
: Group(counters->GetSubgroup("subsystem", "levels"))
@@ -463,6 +474,7 @@ public:
463474
, Level9to16(Group, "level", "9..16")
464475
, Level17(Group, "level", "17")
465476
, Level18(Group, "level", "18")
477+
, Level19(Group, "level", "19")
466478
{}
467479
};
468480

@@ -715,4 +727,3 @@ public:
715727

716728
} // NMonGroup
717729
} // NKikimr
718-

‎ydb/core/blobstorage/vdisk/hulldb/generic/hullds_idx.h

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -200,13 +200,15 @@ namespace NKikimr {
200200
// CompactedLsn. We use CompactedLsn at recovery to determine what logs records
201201
// have to be ignored
202202
TCompactedLsn CompactedLsn;
203+
// DataInplaced aggregated from all levels
204+
ui64 AllLevelsDataInplaced = 0;
203205
// Takes snapshot of the database, actorSystem must be provided, if actorSystem is null,
204206
// optimization applies;
205207
// This function is private and must not be called directly
206208
TLevelIndexSnapshot PrivateGetSnapshot(TActorSystem *actorSystemToNotifyLevelIndex) {
207209
Y_DEBUG_ABORT_UNLESS(Loaded);
208210
return TLevelIndexSnapshot(CurSlice, Fresh.GetSnapshot(), CurSlice->Level0CurSstsNum(),
209-
actorSystemToNotifyLevelIndex, DelayedCompactionDeleterInfo);
211+
AllLevelsDataInplaced, actorSystemToNotifyLevelIndex, DelayedCompactionDeleterInfo);
210212
}
211213

212214
public:
@@ -463,15 +465,19 @@ namespace NKikimr {
463465
ui64 DataHuge = 0;
464466
};
465467

466-
TLevelGroupInfo level0, level1to8, level9to16, level17, level18;
468+
TLevelGroupInfo level0, level1to8, level9to16, level17, level18, level19;
467469

468-
auto process = [](TLevelGroupInfo *stat, TLevelSegment *seg) {
470+
ui64 allLevelsDataInplaced = 0;
471+
472+
auto process = [&allLevelsDataInplaced](TLevelGroupInfo *stat, TLevelSegment *seg) {
469473
stat->SstNum += 1;
470474
stat->NumItems += seg->Info.Items;
471475
stat->NumItemsInplaced += seg->Info.ItemsWithInplacedData;
472476
stat->NumItemsHuge += seg->Info.ItemsWithHugeData;
473477
stat->DataInplaced += seg->Info.InplaceDataTotalSize;
474478
stat->DataHuge += seg->Info.HugeDataTotalSize;
479+
480+
allLevelsDataInplaced += seg->Info.InplaceDataTotalSize;
475481
};
476482

477483
for (const auto& seg : CurSlice->Level0.Segs->Segments) {
@@ -511,6 +517,10 @@ namespace NKikimr {
511517
case 18:
512518
info = &level18;
513519
break;
520+
521+
case 19:
522+
info = &level19;
523+
break;
514524
}
515525

516526
if (info) {
@@ -522,11 +532,14 @@ namespace NKikimr {
522532
++levelIndex;
523533
}
524534

535+
AllLevelsDataInplaced = allLevelsDataInplaced;
536+
525537
for (const auto& p : {std::make_pair(&level0, &stat.Level0),
526538
std::make_pair(&level1to8, &stat.Level1to8),
527539
std::make_pair(&level9to16, &stat.Level9to16),
528540
std::make_pair(&level17, &stat.Level17),
529-
std::make_pair(&level18, &stat.Level18)}) {
541+
std::make_pair(&level18, &stat.Level18),
542+
std::make_pair(&level19, &stat.Level19)}) {
530543
TLevelGroupInfo *from = p.first;
531544
NMonGroup::TLsmLevelGroup *to = p.second;
532545
to->SstNum() = from->SstNum;
@@ -584,4 +597,3 @@ namespace NKikimr {
584597
extern template class TLevelIndex<TKeyBlock, TMemRecBlock>;
585598

586599
} // NKikimr
587-

‎ydb/core/blobstorage/vdisk/hulldb/generic/hullds_idxsnap.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,11 @@ namespace NKikimr {
2828

2929
// TLevelIndexSnapshot is created via TLevelIndex::GetSnapshot
3030
TLevelIndexSnapshot(const TLevelSlicePtr &slice, TFreshDataSnapshot &&freshSnap, ui32 level0SegsNum,
31-
TActorSystem *actorSystem, TIntrusivePtr<TDelayedCompactionDeleterInfo> deleterInfo)
31+
ui64 allLevelsDataInplaced, TActorSystem *actorSystem,
32+
TIntrusivePtr<TDelayedCompactionDeleterInfo> deleterInfo)
3233
: SliceSnap(slice, level0SegsNum)
3334
, FreshSnap(std::move(freshSnap))
35+
, AllLevelsDataInplaced(allLevelsDataInplaced)
3436
, Notifier(actorSystem
3537
? new TDelayedCompactionDeleterNotifier(actorSystem, std::move(deleterInfo))
3638
: nullptr)
@@ -59,6 +61,8 @@ namespace NKikimr {
5961
TLevelSliceSnapshot SliceSnap;
6062
TFreshDataSnapshot FreshSnap;
6163

64+
ui64 AllLevelsDataInplaced = 0;
65+
6266
private:
6367
TIntrusivePtr<TDelayedCompactionDeleterNotifier> Notifier;
6468
};

‎ydb/core/blobstorage/vdisk/skeleton/skeleton_overload_handler.cpp

Lines changed: 75 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -179,59 +179,95 @@ namespace NKikimr {
179179

180180
class TThrottlingController {
181181
private:
182+
NMonGroup::TSkeletonOverloadGroup& Mon;
183+
182184
struct TControls {
185+
TControlWrapper DeviceSpeed;
183186
TControlWrapper MinSstCount;
184187
TControlWrapper MaxSstCount;
185-
TControlWrapper DeviceSpeed;
188+
TControlWrapper MinInplacedSize;
189+
TControlWrapper MaxInplacedSize;
186190

187191
TControls()
188-
: MinSstCount(16, 1, 200)
189-
, MaxSstCount(64, 1, 200)
190-
, DeviceSpeed(50 << 20, 1 << 20, 1 << 30)
192+
: DeviceSpeed(50 << 20, 1 << 20, 1 << 30)
193+
, MinSstCount(100, 1, 1000)
194+
, MaxSstCount(250, 1, 1000)
195+
, MinInplacedSize(20ull << 30, 0, 500ull < 30)
196+
, MaxInplacedSize(60ull << 30, 0, 500ull < 30)
191197
{}
192198

193199
void Register(TIntrusivePtr<TControlBoard> icb) {
200+
icb->RegisterSharedControl(DeviceSpeed, "VDiskControls.ThrottlingDeviceSpeed");
194201
icb->RegisterSharedControl(MinSstCount, "VDiskControls.ThrottlingMinSstCount");
195202
icb->RegisterSharedControl(MaxSstCount, "VDiskControls.ThrottlingMaxSstCount");
196-
icb->RegisterSharedControl(DeviceSpeed, "VDiskControls.ThrottlingDeviceSpeed");
203+
icb->RegisterSharedControl(MinInplacedSize, "VDiskControls.ThrottlingMinInplacedSize");
204+
icb->RegisterSharedControl(MaxInplacedSize, "VDiskControls.ThrottlingMaxInplacedSize");
197205
}
198206
};
199207
TControls Controls;
200208

201209
ui64 CurrentSstCount = 0;
210+
ui64 CurrentInplacedSize = 0;
202211

203212
TInstant CurrentTime;
213+
ui64 CurrentSpeedLimit = 0;
204214
ui64 AvailableBytes = 0;
205215

206-
ui64 GetCurrentSpeedLimit() const {
207-
ui64 minSstCount = (ui64)Controls.MinSstCount;
208-
ui64 maxSstCount = (ui64)Controls.MaxSstCount;
209-
ui64 deviceSpeed = (ui64)Controls.DeviceSpeed;
210-
211-
if (maxSstCount <= minSstCount) {
212-
return deviceSpeed;
216+
private:
217+
static ui64 LinearInterpolation(ui64 curX, ui64 minX, ui64 maxX, ui64 fromY) {
218+
if (maxX <= minX) {
219+
return fromY;
213220
}
214-
if (CurrentSstCount <= minSstCount) {
215-
return deviceSpeed;
221+
if (curX <= minX) {
222+
return fromY;
216223
}
217-
if (CurrentSstCount >= maxSstCount) {
224+
if (curX >= maxX) {
218225
return 0;
219226
}
220-
return (double)(maxSstCount - CurrentSstCount) * deviceSpeed / (maxSstCount - minSstCount);
227+
return (double)(maxX - curX) * fromY / (maxX - minX);
228+
}
229+
230+
ui64 CalcSstCountSpeedLimit() const {
231+
ui64 deviceSpeed = (ui64)Controls.DeviceSpeed;
232+
ui64 minSstCount = (ui64)Controls.MinSstCount;
233+
ui64 maxSstCount = (ui64)Controls.MaxSstCount;
234+
235+
return LinearInterpolation(CurrentSstCount, minSstCount, maxSstCount, deviceSpeed);
236+
}
237+
238+
ui64 CalcInplacedSizeSpeedLimit() const {
239+
ui64 deviceSpeed = (ui64)Controls.DeviceSpeed;
240+
ui64 minInplacedSize = (ui64)Controls.MinInplacedSize;
241+
ui64 maxInplacedSize = (ui64)Controls.MaxInplacedSize;
242+
243+
return LinearInterpolation(CurrentInplacedSize, minInplacedSize, maxInplacedSize, deviceSpeed);
244+
}
245+
246+
ui64 CalcCurrentSpeedLimit() const {
247+
ui64 sstCountSpeedLimit = CalcSstCountSpeedLimit();
248+
ui64 inplacedSizeSpeedLimit = CalcInplacedSizeSpeedLimit();
249+
return std::min(sstCountSpeedLimit, inplacedSizeSpeedLimit);
221250
}
222251

223252
public:
253+
explicit TThrottlingController(NMonGroup::TSkeletonOverloadGroup& mon)
254+
: Mon(mon)
255+
{}
256+
224257
void RegisterIcbControls(TIntrusivePtr<TControlBoard> icb) {
225258
Controls.Register(icb);
226259
}
227260

228261
bool IsActive() const {
229262
ui64 minSstCount = (ui64)Controls.MinSstCount;
230-
return CurrentSstCount > minSstCount;
263+
ui64 minInplacedSize = (ui64)Controls.MinInplacedSize;
264+
265+
return CurrentSstCount > minSstCount ||
266+
CurrentInplacedSize > minInplacedSize;
231267
}
232268

233269
TDuration BytesToDuration(ui64 bytes) const {
234-
auto limit = GetCurrentSpeedLimit();
270+
auto limit = CurrentSpeedLimit;
235271
if (limit == 0) {
236272
return TDuration::Seconds(1);
237273
}
@@ -247,26 +283,38 @@ namespace NKikimr {
247283
return AvailableBytes;
248284
}
249285

250-
void UpdateState(TInstant now, ui64 sstCount) {
286+
void UpdateState(TInstant now, ui64 sstCount, ui64 inplacedSize) {
251287
bool prevActive = IsActive();
252288

253289
CurrentSstCount = sstCount;
290+
Mon.ThrottlingLevel0SstCount() = sstCount;
291+
292+
CurrentInplacedSize = inplacedSize;
293+
Mon.ThrottlingAllLevelsInplacedSize() = inplacedSize;
294+
295+
Mon.ThrottlingIsActive() = (ui64)IsActive();
254296

255297
if (!IsActive()) {
256298
CurrentTime = {};
257299
AvailableBytes = 0;
258-
} else if (!prevActive) {
259-
CurrentTime = now;
260-
AvailableBytes = 0;
300+
CurrentSpeedLimit = (ui64)Controls.DeviceSpeed;
301+
} else {
302+
if (!prevActive) {
303+
CurrentTime = now;
304+
AvailableBytes = 0;
305+
}
306+
CurrentSpeedLimit = CalcCurrentSpeedLimit();
261307
}
308+
309+
Mon.ThrottlingCurrentSpeedLimit() = CurrentSpeedLimit;
262310
}
263311

264312
void UpdateTime(TInstant now) {
265313
if (now <= CurrentTime) {
266314
return;
267315
}
268316
auto us = (now - CurrentTime).MicroSeconds();
269-
AvailableBytes += GetCurrentSpeedLimit() * us / 1000000; // overflow ?
317+
AvailableBytes += CurrentSpeedLimit * us / 1000000;
270318
ui64 deviceSpeed = (ui64)Controls.DeviceSpeed;
271319
AvailableBytes = Min(AvailableBytes, deviceSpeed);
272320
CurrentTime = now;
@@ -292,7 +340,7 @@ namespace NKikimr {
292340
, EmergencyQueue(new TEmergencyQueue(Mon, std::move(vMovedPatch), std::move(vPatchStart), std::move(vput),
293341
std::move(vMultiPut), std::move(loc), std::move(aoput)))
294342
, DynamicPDiskWeightsManager(std::make_shared<TDynamicPDiskWeightsManager>(vctx, pdiskCtx))
295-
, ThrottlingController(new TThrottlingController)
343+
, ThrottlingController(new TThrottlingController(Mon))
296344
{}
297345

298346
TOverloadHandler::~TOverloadHandler() {}
@@ -333,10 +381,12 @@ namespace NKikimr {
333381
auto snapshot = Hull->GetSnapshot(); // THullDsSnap
334382
auto& logoBlobsSnap = snapshot.LogoBlobsSnap; // TLogoBlobsSnapshot
335383
auto& sliceSnap = logoBlobsSnap.SliceSnap; // TLevelSliceSnapshot
384+
336385
auto sstCount = sliceSnap.GetLevel0SstsNum();
386+
auto dataInplacedSize = logoBlobsSnap.AllLevelsDataInplaced;
337387

338388
auto now = ctx.Now();
339-
ThrottlingController->UpdateState(now, sstCount);
389+
ThrottlingController->UpdateState(now, sstCount, dataInplacedSize);
340390

341391
if (ThrottlingController->IsActive()) {
342392
ThrottlingController->UpdateTime(now);

0 commit comments

Comments
 (0)