Skip to content

Commit 77e599f

Browse files
kungakunga
authored
Set KQP & Resource Broker limits from MemoryController (#7590)
1 parent 833d136 commit 77e599f

File tree

11 files changed

+310
-156
lines changed

11 files changed

+310
-156
lines changed

‎ydb/core/memory_controller/memory_controller.cpp

Lines changed: 81 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#include "memory_controller.h"
2+
#include "memory_controller_config.h"
23
#include "memtable_collection.h"
34
#include <ydb/core/base/counters.h>
45
#include <ydb/core/base/memory_controller_iface.h>
@@ -8,6 +9,7 @@
89
#include <ydb/core/node_whiteboard/node_whiteboard.h>
910
#include <ydb/core/protos/memory_controller_config.pb.h>
1011
#include <ydb/core/protos/memory_stats.pb.h>
12+
#include <ydb/core/tablet/resource_broker.h>
1113
#include <ydb/core/tablet_flat/shared_sausagecache.h>
1214
#include <ydb/library/actors/core/actor_bootstrapped.h>
1315
#include <ydb/library/actors/core/log.h>
@@ -27,8 +29,23 @@ ui64 SafeDiff(ui64 a, ui64 b) {
2729
namespace {
2830

2931
using namespace NActors;
32+
using namespace NResourceBroker;
3033
using TCounterPtr = ::NMonitoring::TDynamicCounters::TCounterPtr;
3134

35+
struct TResourceBrokerLimits {
36+
ui64 LimitBytes;
37+
ui64 QueryExecutionLimitBytes;
38+
39+
auto operator<=>(const TResourceBrokerLimits&) const = default;
40+
41+
TString ToString() const noexcept {
42+
TStringBuilder result;
43+
result << "LimitBytes: " << LimitBytes;
44+
result << " QueryExecutionLimitBytes: " << QueryExecutionLimitBytes;
45+
return result;
46+
}
47+
};
48+
3249
class TMemoryConsumer : public IMemoryConsumer {
3350
public:
3451
TMemoryConsumer(EMemoryConsumerKind kind, TActorId actorId)
@@ -52,27 +69,18 @@ class TMemoryConsumer : public IMemoryConsumer {
5269
std::atomic<ui64> Consumption = 0;
5370
};
5471

55-
struct TConsumerConfig {
56-
std::optional<float> MinPercent;
57-
std::optional<ui64> MinBytes;
58-
std::optional<float> MaxPercent;
59-
std::optional<ui64> MaxBytes;
60-
bool CanZeroLimit = false;
61-
};
62-
6372
struct TConsumerState {
6473
const EMemoryConsumerKind Kind;
6574
const TActorId ActorId;
6675
const ui64 Consumption;
67-
const TConsumerConfig Config;
6876
ui64 MinBytes = 0;
6977
ui64 MaxBytes = 0;
78+
bool CanZeroLimit = false;
7079

71-
TConsumerState(const TMemoryConsumer& consumer, TConsumerConfig config)
80+
TConsumerState(const TMemoryConsumer& consumer)
7281
: Kind(consumer.Kind)
7382
, ActorId(consumer.ActorId)
7483
, Consumption(consumer.GetConsumption())
75-
, Config(config)
7684
{
7785
}
7886

@@ -132,6 +140,8 @@ class TMemoryController : public TActorBootstrapped<TMemoryController> {
132140
HFunc(TEvMemTableRegister, Handle);
133141
HFunc(TEvMemTableUnregister, Handle);
134142
HFunc(TEvMemTableCompacted, Handle);
143+
144+
HFunc(TEvResourceBroker::TEvConfigureResult, Handle);
135145
}
136146
}
137147

@@ -144,9 +154,10 @@ class TMemoryController : public TActorBootstrapped<TMemoryController> {
144154
auto processMemoryInfo = ProcessMemoryInfoProvider->Get();
145155

146156
bool hasMemTotalHardLimit = false;
147-
ui64 hardLimitBytes = GetHardLimitBytes(processMemoryInfo, hasMemTotalHardLimit);
148-
ui64 softLimitBytes = GetSoftLimitBytes(hardLimitBytes);
149-
ui64 targetUtilizationBytes = GetTargetUtilizationBytes(hardLimitBytes);
157+
ui64 hardLimitBytes = GetHardLimitBytes(Config, processMemoryInfo, hasMemTotalHardLimit);
158+
ui64 softLimitBytes = GetSoftLimitBytes(Config, hardLimitBytes);
159+
ui64 targetUtilizationBytes = GetTargetUtilizationBytes(Config, hardLimitBytes);
160+
ui64 activitiesLimitBytes = GetActivitiesLimitBytes(Config, hardLimitBytes);
150161

151162
TVector<TConsumerState> consumers(::Reserve(Consumers.size()));
152163
ui64 consumersConsumption = 0;
@@ -189,6 +200,7 @@ class TMemoryController : public TActorBootstrapped<TMemoryController> {
189200
<< " MemTotal: " << processMemoryInfo.MemTotal << " MemAvailable: " << processMemoryInfo.MemAvailable
190201
<< " AllocatedMemory: " << processMemoryInfo.AllocatedMemory << " AllocatorCachesMemory: " << processMemoryInfo.AllocatorCachesMemory
191202
<< " HardLimit: " << hardLimitBytes << " SoftLimit: " << softLimitBytes << " TargetUtilization: " << targetUtilizationBytes
203+
<< " ActivitiesLimitBytes: " << activitiesLimitBytes
192204
<< " ConsumersConsumption: " << consumersConsumption << " OtherConsumption: " << otherConsumption << " ExternalConsumption: " << externalConsumption
193205
<< " TargetConsumersConsumption: " << targetConsumersConsumption << " ResultingConsumersConsumption: " << resultingConsumersConsumption
194206
<< " Coefficient: " << coefficient);
@@ -202,6 +214,7 @@ class TMemoryController : public TActorBootstrapped<TMemoryController> {
202214
Counters->GetCounter("Stats/HardLimit")->Set(hardLimitBytes);
203215
Counters->GetCounter("Stats/SoftLimit")->Set(softLimitBytes);
204216
Counters->GetCounter("Stats/TargetUtilization")->Set(targetUtilizationBytes);
217+
Counters->GetCounter("Stats/ActivitiesLimitBytes")->Set(activitiesLimitBytes);
205218
Counters->GetCounter("Stats/ConsumersConsumption")->Set(consumersConsumption);
206219
Counters->GetCounter("Stats/OtherConsumption")->Set(otherConsumption);
207220
Counters->GetCounter("Stats/ExternalConsumption")->Set(externalConsumption);
@@ -227,7 +240,7 @@ class TMemoryController : public TActorBootstrapped<TMemoryController> {
227240
ui64 consumersLimitBytes = 0;
228241
for (const auto& consumer : consumers) {
229242
ui64 limitBytes = consumer.GetLimit(coefficient);
230-
if (resultingConsumersConsumption + otherConsumption + externalConsumption > softLimitBytes && consumer.Config.CanZeroLimit) {
243+
if (resultingConsumersConsumption + otherConsumption + externalConsumption > softLimitBytes && consumer.CanZeroLimit) {
231244
limitBytes = SafeDiff(limitBytes, resultingConsumersConsumption + otherConsumption + externalConsumption - softLimitBytes);
232245
}
233246
consumersLimitBytes += limitBytes;
@@ -249,6 +262,9 @@ class TMemoryController : public TActorBootstrapped<TMemoryController> {
249262
Counters->GetCounter("Stats/ConsumersLimit")->Set(consumersLimitBytes);
250263
memoryStats.SetConsumersLimit(consumersLimitBytes);
251264

265+
// Note: for now ResourceBroker and its queues aren't MemoryController consumers and don't share limits with other caches
266+
ApplyResourceBrokerLimits(hardLimitBytes, activitiesLimitBytes);
267+
252268
Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(SelfId().NodeId()), memoryStatsUpdate);
253269

254270
ctx.Schedule(Interval, new TEvents::TEvWakeup());
@@ -283,6 +299,14 @@ class TMemoryController : public TActorBootstrapped<TMemoryController> {
283299
}
284300
}
285301

302+
void Handle(TEvResourceBroker::TEvConfigureResult::TPtr &ev, const TActorContext& ctx) {
303+
const auto *msg = ev->Get();
304+
LOG_LOG_S(ctx,
305+
msg->Record.GetSuccess() ? NActors::NLog::PRI_INFO : NActors::NLog::PRI_ERROR,
306+
NKikimrServices::MEMORY_CONTROLLER,
307+
"ResourceBroker configure result " << msg->Record.ShortDebugString());
308+
}
309+
286310
double BinarySearchCoefficient(const TVector<TConsumerState>& consumers, ui64 availableMemory) {
287311
static const ui32 BinarySearchIterations = 20;
288312

@@ -326,6 +350,36 @@ class TMemoryController : public TActorBootstrapped<TMemoryController> {
326350
}
327351
}
328352

353+
void ApplyResourceBrokerLimits(ui64 hardLimitBytes, ui64 activitiesLimitBytes) {
354+
ui64 queryExecutionLimitBytes = GetQueryExecutionLimitBytes(Config, hardLimitBytes);
355+
356+
TResourceBrokerLimits newLimits{
357+
activitiesLimitBytes,
358+
queryExecutionLimitBytes
359+
};
360+
361+
if (newLimits == CurrentResourceBrokerLimits) {
362+
return;
363+
}
364+
365+
CurrentResourceBrokerLimits = newLimits;
366+
367+
LOG_INFO_S(TlsActivationContext->AsActorContext(), NKikimrServices::MEMORY_CONTROLLER, "Consumer QueryExecution state:"
368+
<< " Limit: " << newLimits.QueryExecutionLimitBytes);
369+
370+
Counters->GetCounter("Consumer/QueryExecution/Limit")->Set(newLimits.QueryExecutionLimitBytes);
371+
372+
TAutoPtr<TEvResourceBroker::TEvConfigure> configure = new TEvResourceBroker::TEvConfigure();
373+
configure->Merge = true;
374+
configure->Record.MutableResourceLimit()->SetMemory(activitiesLimitBytes);
375+
376+
auto queue = configure->Record.AddQueues();
377+
queue->SetName(NLocalDb::KqpResourceManagerQueue);
378+
queue->MutableLimit()->SetMemory(queryExecutionLimitBytes);
379+
380+
Send(MakeResourceBrokerID(), configure.Release());
381+
}
382+
329383
TConsumerCounters& GetConsumerCounters(EMemoryConsumerKind consumer) {
330384
auto it = ConsumerCounters.FindPtr(consumer);
331385
if (it) {
@@ -358,128 +412,30 @@ class TMemoryController : public TActorBootstrapped<TMemoryController> {
358412
}
359413
}
360414

361-
TConsumerState BuildConsumerState(const TMemoryConsumer& consumer, ui64 availableMemory) const {
362-
auto config = GetConsumerConfig(consumer.Kind);
415+
TConsumerState BuildConsumerState(const TMemoryConsumer& consumer, ui64 hardLimitBytes) const {
416+
TConsumerState result(consumer);
363417

364-
std::optional<ui64> minBytes;
365-
std::optional<ui64> maxBytes;
366-
367-
if (config.MinPercent.has_value() && config.MinBytes.has_value()) {
368-
minBytes = Max(GetPercent(config.MinPercent.value(), availableMemory), config.MinBytes.value());
369-
} else if (config.MinPercent.has_value()) {
370-
minBytes = GetPercent(config.MinPercent.value(), availableMemory);
371-
} else if (config.MinBytes.has_value()) {
372-
minBytes = config.MinBytes.value();
373-
}
374-
375-
if (config.MaxPercent.has_value() && config.MaxBytes.has_value()) {
376-
maxBytes = Min(GetPercent(config.MaxPercent.value(), availableMemory), config.MaxBytes.value());
377-
} else if (config.MaxPercent.has_value()) {
378-
maxBytes = GetPercent(config.MaxPercent.value(), availableMemory);
379-
} else if (config.MaxBytes.has_value()) {
380-
maxBytes = config.MaxBytes.value();
381-
}
382-
383-
if (minBytes.has_value() && !maxBytes.has_value()) {
384-
maxBytes = minBytes;
385-
}
386-
if (!minBytes.has_value() && maxBytes.has_value()) {
387-
minBytes = maxBytes;
388-
}
389-
390-
TConsumerState result(std::move(consumer), config);
391-
392-
result.MinBytes = minBytes.value_or(0);
393-
result.MaxBytes = maxBytes.value_or(0);
394-
if (result.MinBytes > result.MaxBytes) {
395-
result.MinBytes = result.MaxBytes;
396-
}
397-
398-
return result;
399-
}
400-
401-
TConsumerConfig GetConsumerConfig(EMemoryConsumerKind consumer) const {
402-
TConsumerConfig result;
403-
404-
switch (consumer) {
418+
switch (consumer.Kind) {
405419
case EMemoryConsumerKind::MemTable: {
406-
if (Config.HasMemTableMinPercent() || Config.GetMemTableMinPercent()) {
407-
result.MinPercent = Config.GetMemTableMinPercent();
408-
}
409-
if (Config.HasMemTableMinBytes() || Config.GetMemTableMinBytes()) {
410-
result.MinBytes = Config.GetMemTableMinBytes();
411-
}
412-
if (Config.HasMemTableMaxPercent() || Config.GetMemTableMaxPercent()) {
413-
result.MaxPercent = Config.GetMemTableMaxPercent();
414-
}
415-
if (Config.HasMemTableMaxBytes() || Config.GetMemTableMaxBytes()) {
416-
result.MaxBytes = Config.GetMemTableMaxBytes();
417-
}
420+
result.MinBytes = GetMemTableMinBytes(Config, hardLimitBytes);
421+
result.MaxBytes = GetMemTableMaxBytes(Config, hardLimitBytes);
418422
break;
419423
}
420424
case EMemoryConsumerKind::SharedCache: {
421-
if (Config.HasSharedCacheMinPercent() || Config.GetSharedCacheMinPercent()) {
422-
result.MinPercent = Config.GetSharedCacheMinPercent();
423-
}
424-
if (Config.HasSharedCacheMinBytes() || Config.GetSharedCacheMinBytes()) {
425-
result.MinBytes = Config.GetSharedCacheMinBytes();
426-
}
427-
if (Config.HasSharedCacheMaxPercent() || Config.GetSharedCacheMaxPercent()) {
428-
result.MaxPercent = Config.GetSharedCacheMaxPercent();
429-
}
430-
if (Config.HasSharedCacheMaxBytes() || Config.GetSharedCacheMaxBytes()) {
431-
result.MaxBytes = Config.GetSharedCacheMaxBytes();
432-
}
425+
result.MinBytes = GetSharedCacheMinBytes(Config, hardLimitBytes);
426+
result.MaxBytes = GetSharedCacheMaxBytes(Config, hardLimitBytes);
433427
result.CanZeroLimit = true;
434428
break;
435429
}
436430
default:
437431
Y_ABORT("Unhandled consumer");
438432
}
439433

440-
return result;
441-
}
442-
443-
ui64 GetHardLimitBytes(const TProcessMemoryInfo& info, bool& hasMemTotalHardLimit) const {
444-
if (Config.HasHardLimitBytes()) {
445-
ui64 hardLimitBytes = Config.GetHardLimitBytes();
446-
if (info.CGroupLimit.has_value()) {
447-
hardLimitBytes = Min(hardLimitBytes, info.CGroupLimit.value());
448-
}
449-
return hardLimitBytes;
450-
}
451-
if (info.CGroupLimit.has_value()) {
452-
return info.CGroupLimit.value();
453-
}
454-
if (info.MemTotal) {
455-
hasMemTotalHardLimit = true;
456-
return info.MemTotal.value();
457-
}
458-
return 512_MB; // fallback
459-
}
460-
461-
ui64 GetSoftLimitBytes(ui64 hardLimitBytes) const {
462-
if (Config.HasSoftLimitPercent() && Config.HasSoftLimitBytes()) {
463-
return Min(GetPercent(Config.GetSoftLimitPercent(), hardLimitBytes), Config.GetSoftLimitBytes());
464-
}
465-
if (Config.HasSoftLimitBytes()) {
466-
return Config.GetSoftLimitBytes();
467-
}
468-
return GetPercent(Config.GetSoftLimitPercent(), hardLimitBytes);
469-
}
470-
471-
ui64 GetTargetUtilizationBytes(ui64 hardLimitBytes) const {
472-
if (Config.HasTargetUtilizationPercent() && Config.HasTargetUtilizationBytes()) {
473-
return Min(GetPercent(Config.GetTargetUtilizationPercent(), hardLimitBytes), Config.GetTargetUtilizationBytes());
474-
}
475-
if (Config.HasTargetUtilizationBytes()) {
476-
return Config.GetTargetUtilizationBytes();
434+
if (result.MinBytes > result.MaxBytes) {
435+
result.MinBytes = result.MaxBytes;
477436
}
478-
return GetPercent(Config.GetTargetUtilizationPercent(), hardLimitBytes);
479-
}
480437

481-
ui64 GetPercent(float percent, ui64 value) const {
482-
return static_cast<ui64>(static_cast<double>(value) * (percent / 100.0));
438+
return result;
483439
}
484440

485441
private:
@@ -490,6 +446,7 @@ class TMemoryController : public TActorBootstrapped<TMemoryController> {
490446
NKikimrConfig::TMemoryControllerConfig Config;
491447
const TIntrusivePtr<::NMonitoring::TDynamicCounters> Counters;
492448
TMap<EMemoryConsumerKind, TConsumerCounters> ConsumerCounters;
449+
std::optional<TResourceBrokerLimits> CurrentResourceBrokerLimits;
493450
};
494451

495452
}
@@ -498,7 +455,7 @@ IActor* CreateMemoryController(
498455
TDuration interval,
499456
TIntrusiveConstPtr<IProcessMemoryInfoProvider> processMemoryInfoProvider,
500457
const NKikimrConfig::TMemoryControllerConfig& config,
501-
TIntrusivePtr<::NMonitoring::TDynamicCounters> counters) {
458+
const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters) {
502459
return new TMemoryController(
503460
interval,
504461
std::move(processMemoryInfoProvider),

‎ydb/core/memory_controller/memory_controller.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,6 @@ NActors::IActor* CreateMemoryController(
1414
TDuration interval,
1515
TIntrusiveConstPtr<IProcessMemoryInfoProvider> processMemoryInfoProvider,
1616
const NKikimrConfig::TMemoryControllerConfig& config,
17-
TIntrusivePtr<::NMonitoring::TDynamicCounters> counters);
17+
const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters);
1818

1919
}

0 commit comments

Comments
 (0)