Skip to content

Commit 70bb381

Browse files
azevaykinazevaykin
authored
25-1: TLI system views (#16103) (#18928)
2 parents 1d5f8c9 + 5418af0 commit 70bb381

File tree

21 files changed

+613
-139
lines changed

21 files changed

+613
-139
lines changed

‎ydb/core/protos/sys_view.proto

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ message TPartitionStats {
4747

4848
optional uint64 ByKeyFilterSize = 21;
4949
optional uint32 FollowerId = 22;
50+
51+
optional uint64 LocksAcquired = 23;
52+
optional uint64 LocksWholeShard = 24;
53+
optional uint64 LocksBroken = 25;
5054
}
5155

5256
message TPartitionStatsResult {
@@ -127,8 +131,10 @@ enum EStatsType {
127131
METRICS_ONE_HOUR = 8;
128132
TOP_REQUEST_UNITS_ONE_MINUTE = 9;
129133
TOP_REQUEST_UNITS_ONE_HOUR = 10;
130-
TOP_PARTITIONS_ONE_MINUTE = 11;
131-
TOP_PARTITIONS_ONE_HOUR = 12;
134+
TOP_PARTITIONS_BY_CPU_ONE_MINUTE = 11;
135+
TOP_PARTITIONS_BY_CPU_ONE_HOUR = 12;
136+
TOP_PARTITIONS_BY_TLI_ONE_MINUTE = 13;
137+
TOP_PARTITIONS_BY_TLI_ONE_HOUR = 14;
132138
}
133139

134140
message TEvGetQueryStats {
@@ -602,6 +608,9 @@ message TTopPartitionsInfo {
602608
optional uint64 IndexSize = 8;
603609
optional uint32 InFlightTxCount = 9;
604610
optional uint32 FollowerId = 10;
611+
optional uint64 LocksAcquired = 11;
612+
optional uint64 LocksWholeShard = 12;
613+
optional uint64 LocksBroken = 13;
605614
}
606615

607616
message TTopPartitionsEntry {
@@ -628,6 +637,7 @@ message TEvGetTopPartitionsResponse {
628637

629638
// partitions stats collector -> SVP
630639
message TEvSendTopPartitions {
631-
repeated TTopPartitionsInfo Partitions = 1;
640+
repeated TTopPartitionsInfo PartitionsByCpu = 1;
641+
repeated TTopPartitionsInfo PartitionsByTli = 3;
632642
optional uint64 TimeUs = 2;
633643
}

‎ydb/core/sys_view/common/schema.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -282,8 +282,10 @@ class TSystemViewResolver : public ISystemViewResolver {
282282
RegisterColumnTableSystemView<Schema::PrimaryIndexGranuleStats>(TablePrimaryIndexGranuleStatsName);
283283
RegisterColumnTableSystemView<Schema::PrimaryIndexOptimizerStats>(TablePrimaryIndexOptimizerStatsName);
284284

285-
RegisterSystemView<Schema::TopPartitions>(TopPartitions1MinuteName);
286-
RegisterSystemView<Schema::TopPartitions>(TopPartitions1HourName);
285+
RegisterSystemView<Schema::TopPartitions>(TopPartitionsByCpu1MinuteName);
286+
RegisterSystemView<Schema::TopPartitions>(TopPartitionsByCpu1HourName);
287+
RegisterSystemView<Schema::TopPartitionsTli>(TopPartitionsByTli1MinuteName);
288+
RegisterSystemView<Schema::TopPartitionsTli>(TopPartitionsByTli1HourName);
287289

288290
RegisterPgTablesSystemViews();
289291

‎ydb/core/sys_view/common/schema.h

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,11 @@ constexpr TStringBuf TablePrimaryIndexPortionStatsName = "primary_index_portion_
4242
constexpr TStringBuf TablePrimaryIndexGranuleStatsName = "primary_index_granule_stats";
4343
constexpr TStringBuf TablePrimaryIndexOptimizerStatsName = "primary_index_optimizer_stats";
4444

45-
constexpr TStringBuf TopPartitions1MinuteName = "top_partitions_one_minute";
46-
constexpr TStringBuf TopPartitions1HourName = "top_partitions_one_hour";
45+
constexpr TStringBuf TopPartitionsByCpu1MinuteName = "top_partitions_one_minute";
46+
constexpr TStringBuf TopPartitionsByCpu1HourName = "top_partitions_one_hour";
47+
48+
constexpr TStringBuf TopPartitionsByTli1MinuteName = "top_partitions_by_tli_one_minute";
49+
constexpr TStringBuf TopPartitionsByTli1HourName = "top_partitions_by_tli_one_hour";
4750

4851
constexpr TStringBuf PgTablesName = "pg_tables";
4952
constexpr TStringBuf InformationSchemaTablesName = "tables";
@@ -90,6 +93,9 @@ struct Schema : NIceDb::Schema {
9093
struct LastTtlRowsProcessed : Column<25, NScheme::NTypeIds::Uint64> {};
9194
struct LastTtlRowsErased : Column<26, NScheme::NTypeIds::Uint64> {};
9295
struct FollowerId : Column<27, NScheme::NTypeIds::Uint32> {};
96+
struct LocksAcquired : Column<28, NScheme::NTypeIds::Uint64> {};
97+
struct LocksWholeShard : Column<29, NScheme::NTypeIds::Uint64> {};
98+
struct LocksBroken : Column<30, NScheme::NTypeIds::Uint64> {};
9399

94100
using TKey = TableKey<OwnerId, PathId, PartIdx, FollowerId>;
95101
using TColumns = TableColumns<
@@ -119,7 +125,11 @@ struct Schema : NIceDb::Schema {
119125
LastTtlRunTime,
120126
LastTtlRowsProcessed,
121127
LastTtlRowsErased,
122-
FollowerId>;
128+
FollowerId,
129+
LocksAcquired,
130+
LocksWholeShard,
131+
LocksBroken
132+
>;
123133
};
124134

125135
struct Nodes : Table<2> {
@@ -717,6 +727,36 @@ struct Schema : NIceDb::Schema {
717727
Rank,
718728
Config>;
719729
};
730+
731+
struct TopPartitionsTli : Table<23> {
732+
struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {};
733+
struct Rank : Column<2, NScheme::NTypeIds::Uint32> {};
734+
struct TabletId : Column<3, NScheme::NTypeIds::Uint64> {};
735+
struct Path : Column<4, NScheme::NTypeIds::Utf8> {};
736+
struct LocksAcquired : Column<5, NScheme::NTypeIds::Uint64> {};
737+
struct LocksWholeShard : Column<6, NScheme::NTypeIds::Uint64> {};
738+
struct LocksBroken : Column<7, NScheme::NTypeIds::Uint64> {};
739+
struct NodeId : Column<8, NScheme::NTypeIds::Uint32> {};
740+
struct DataSize : Column<9, NScheme::NTypeIds::Uint64> {};
741+
struct RowCount : Column<10, NScheme::NTypeIds::Uint64> {};
742+
struct IndexSize : Column<11, NScheme::NTypeIds::Uint64> {};
743+
struct FollowerId : Column<12, NScheme::NTypeIds::Uint32> {};
744+
745+
using TKey = TableKey<IntervalEnd, Rank>;
746+
using TColumns = TableColumns<
747+
IntervalEnd,
748+
Rank,
749+
TabletId,
750+
Path,
751+
LocksAcquired,
752+
LocksWholeShard,
753+
LocksBroken,
754+
NodeId,
755+
DataSize,
756+
RowCount,
757+
IndexSize,
758+
FollowerId>;
759+
};
720760
};
721761

722762
bool MaybeSystemViewPath(const TVector<TString>& path);

‎ydb/core/sys_view/partition_stats/partition_stats.cpp

Lines changed: 104 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ class TPartitionStatsCollector : public TActorBootstrapped<TPartitionStatsCollec
3232
SVLOG_D("NSysView::TPartitionStatsCollector bootstrapped");
3333

3434
if (AppData()->UsePartitionStatsCollectorForTests) {
35-
OverloadedPartitionBound = 0.0;
35+
OverloadedByCpuPartitionBound = 0.0;
36+
OverloadedByTliPartitionBound = 0;
3637
ProcessOverloadedInterval = TDuration::Seconds(1);
3738
}
3839

@@ -88,25 +89,32 @@ class TPartitionStatsCollector : public TActorBootstrapped<TPartitionStatsCollec
8889

8990
auto& oldPartitions = table.Partitions;
9091
std::unordered_map<TShardIdx, TPartitionStats> newPartitions;
91-
std::set<TOverloadedFollower> overloaded;
92+
std::set<TFollowerStats> overloadedByCpu, overloadedByTli;
9293

9394
for (auto shardIdx : ev->Get()->ShardIndices) {
9495
auto old = oldPartitions.find(shardIdx);
9596
if (old != oldPartitions.end()) {
9697
newPartitions[shardIdx] = old->second;
9798

9899
for (const auto& followerStat: old->second.FollowerStats) {
99-
if (IsPartitionOverloaded(followerStat.second))
100-
overloaded.insert({shardIdx, followerStat.first});
100+
if (IsPartitionOverloadedByCpu(followerStat.second))
101+
overloadedByCpu.insert({shardIdx, followerStat.first});
102+
if (IsPartitionOverloadedByTli(followerStat.second))
103+
overloadedByTli.insert({shardIdx, followerStat.first});
101104
}
102105
}
103106
}
104107

105-
if (!overloaded.empty()) {
106-
tables.Overloaded[pathId].swap(overloaded);
108+
if (!overloadedByCpu.empty()) {
109+
tables.OverloadedByCpu[pathId].swap(overloadedByCpu);
107110
} else {
108-
tables.Overloaded.erase(pathId);
111+
tables.OverloadedByCpu.erase(pathId);
109112
}
113+
if (!overloadedByTli.empty()) {
114+
tables.OverloadedByTli[pathId].swap(overloadedByTli);
115+
} else {
116+
tables.OverloadedByTli.erase(pathId);
117+
}
110118

111119
oldPartitions.swap(newPartitions);
112120
table.ShardIndices.swap(ev->Get()->ShardIndices);
@@ -125,7 +133,8 @@ class TPartitionStatsCollector : public TActorBootstrapped<TPartitionStatsCollec
125133

126134
auto& tables = DomainTables[domainKey];
127135
tables.Stats.erase(pathId);
128-
tables.Overloaded.erase(pathId);
136+
tables.OverloadedByCpu.erase(pathId);
137+
tables.OverloadedByTli.erase(pathId);
129138
}
130139

131140
void Handle(TEvSysView::TEvSendPartitionStats::TPtr& ev) {
@@ -153,18 +162,29 @@ class TPartitionStatsCollector : public TActorBootstrapped<TPartitionStatsCollec
153162

154163
auto& followerStats = partitionStats.FollowerStats[followerId];
155164

156-
TOverloadedFollower overloadedFollower = {shardIdx, followerId};
157-
if (IsPartitionOverloaded(newStats)) {
158-
tables.Overloaded[pathId].insert(overloadedFollower);
165+
TFollowerStats overloadedFollower = {shardIdx, followerId};
166+
if (IsPartitionOverloadedByCpu(newStats)) {
167+
tables.OverloadedByCpu[pathId].insert(overloadedFollower);
159168
} else {
160-
auto overloadedFound = tables.Overloaded.find(pathId);
161-
if (overloadedFound != tables.Overloaded.end()) {
169+
auto overloadedFound = tables.OverloadedByCpu.find(pathId);
170+
if (overloadedFound != tables.OverloadedByCpu.end()) {
162171
overloadedFound->second.erase(overloadedFollower);
163172
if (overloadedFound->second.empty()) {
164-
tables.Overloaded.erase(pathId);
173+
tables.OverloadedByCpu.erase(pathId);
165174
}
166175
}
167176
}
177+
if (IsPartitionOverloadedByTli(newStats)) {
178+
tables.OverloadedByTli[pathId].insert(overloadedFollower);
179+
} else {
180+
auto overloadedFound = tables.OverloadedByTli.find(pathId);
181+
if (overloadedFound != tables.OverloadedByTli.end()) {
182+
overloadedFound->second.erase(overloadedFollower);
183+
if (overloadedFound->second.empty()) {
184+
tables.OverloadedByTli.erase(pathId);
185+
}
186+
}
187+
}
168188

169189
if (followerStats.HasTtlStats()) {
170190
newStats.MutableTtlStats()->Swap(followerStats.MutableTtlStats());
@@ -376,37 +396,54 @@ class TPartitionStatsCollector : public TActorBootstrapped<TPartitionStatsCollec
376396
}
377397
auto& domainTables = domainFound->second;
378398

379-
struct TPartition {
399+
struct TPartitionByCpu {
380400
TPathId PathId;
381401
TShardIdx ShardIdx;
382402
ui32 FollowerId;
383403
double CPUCores;
384404
};
385-
std::vector<TPartition> sorted;
405+
std::vector<TPartitionByCpu> sortedByCpu;
386406

387-
for (const auto& [pathId, overloadedFollowers] : domainTables.Overloaded) {
388-
for (const TOverloadedFollower& overloadedFollower : overloadedFollowers) {
407+
struct TPartitionByTli {
408+
TPathId PathId;
409+
TShardIdx ShardIdx;
410+
ui32 FollowerId;
411+
ui64 LocksBroken;
412+
};
413+
std::vector<TPartitionByTli> sortedByTli;
414+
415+
for (const auto& [pathId, overloadedFollowers] : domainTables.OverloadedByCpu) {
416+
for (const TFollowerStats& overloadedFollower : overloadedFollowers) {
389417
const auto& table = domainTables.Stats[pathId];
390418
const auto& partition = table.Partitions.at(overloadedFollower.ShardIdx).FollowerStats.at(overloadedFollower.FollowerId);
391-
sorted.emplace_back(TPartition{pathId, overloadedFollower.ShardIdx, overloadedFollower.FollowerId, partition.GetCPUCores()});
419+
sortedByCpu.emplace_back(TPartitionByCpu{pathId, overloadedFollower.ShardIdx, overloadedFollower.FollowerId, partition.GetCPUCores()});
392420
}
393421
}
422+
for (const auto& [pathId, overloadedFollowers] : domainTables.OverloadedByTli) {
423+
for (const TFollowerStats& overloadedFollower : overloadedFollowers) {
424+
const auto& table = domainTables.Stats[pathId];
425+
const auto& partition = table.Partitions.at(overloadedFollower.ShardIdx).FollowerStats.at(overloadedFollower.FollowerId);
426+
sortedByTli.emplace_back(TPartitionByTli{pathId, overloadedFollower.ShardIdx, overloadedFollower.FollowerId, partition.GetLocksBroken()});
427+
}
428+
}
394429

395-
std::sort(sorted.begin(), sorted.end(),
430+
std::sort(sortedByCpu.begin(), sortedByCpu.end(),
396431
[] (const auto& l, const auto& r) { return l.CPUCores > r.CPUCores; });
432+
std::sort(sortedByTli.begin(), sortedByTli.end(),
433+
[] (const auto& l, const auto& r) { return l.LocksBroken > r.LocksBroken; });
397434

398435
auto now = TActivationContext::Now();
399436
auto nowUs = now.MicroSeconds();
400437

401438
size_t count = 0;
402439
auto sendEvent = MakeHolder<TEvSysView::TEvSendTopPartitions>();
403-
for (const auto& entry : sorted) {
440+
for (const auto& entry : sortedByCpu) {
404441
const auto& table = domainTables.Stats[entry.PathId];
405442
const auto& followerStats = table.Partitions.at(entry.ShardIdx).FollowerStats;
406443
const auto& partition = followerStats.at(entry.FollowerId);
407444
const auto& leaderPartition = followerStats.at(0);
408445

409-
auto* result = sendEvent->Record.AddPartitions();
446+
auto* result = sendEvent->Record.AddPartitionsByCpu();
410447
result->SetTabletId(partition.GetTabletId());
411448
result->SetPath(table.Path);
412449
result->SetPeakTimeUs(nowUs);
@@ -422,11 +459,34 @@ class TPartitionStatsCollector : public TActorBootstrapped<TPartitionStatsCollec
422459
break;
423460
}
424461
}
462+
for (const auto& entry : sortedByTli) {
463+
const auto& table = domainTables.Stats[entry.PathId];
464+
const auto& followerStats = table.Partitions.at(entry.ShardIdx).FollowerStats;
465+
const auto& partition = followerStats.at(entry.FollowerId);
466+
const auto& leaderPartition = followerStats.at(0);
467+
468+
auto* result = sendEvent->Record.AddPartitionsByTli();
469+
result->SetTabletId(partition.GetTabletId());
470+
result->SetPath(table.Path);
471+
result->SetLocksAcquired(partition.GetLocksAcquired());
472+
result->SetLocksWholeShard(partition.GetLocksWholeShard());
473+
result->SetLocksBroken(partition.GetLocksBroken());
474+
result->SetNodeId(partition.GetNodeId());
475+
result->SetDataSize(leaderPartition.GetDataSize());
476+
result->SetRowCount(leaderPartition.GetRowCount());
477+
result->SetIndexSize(leaderPartition.GetIndexSize());
478+
result->SetFollowerId(partition.GetFollowerId());
479+
480+
if (++count == TOP_PARTITIONS_COUNT) {
481+
break;
482+
}
483+
}
425484

426485
sendEvent->Record.SetTimeUs(nowUs);
427486

428487
SVLOG_D("NSysView::TPartitionStatsCollector: TEvProcessOverloaded "
429-
<< "top size# " << sorted.size()
488+
<< ", top size by CPU # " << sortedByCpu.size()
489+
<< ", top size by TLI # " << sortedByTli.size()
430490
<< ", time# " << now);
431491

432492
Send(MakePipePerNodeCacheID(false),
@@ -447,8 +507,11 @@ class TPartitionStatsCollector : public TActorBootstrapped<TPartitionStatsCollec
447507
TBase::PassAway();
448508
}
449509

450-
bool IsPartitionOverloaded(const NKikimrSysView::TPartitionStats& stats) const {
451-
return stats.GetCPUCores() >= OverloadedPartitionBound;
510+
bool IsPartitionOverloadedByCpu(const NKikimrSysView::TPartitionStats& stats) const {
511+
return stats.GetCPUCores() >= OverloadedByCpuPartitionBound;
512+
}
513+
bool IsPartitionOverloadedByTli(const NKikimrSysView::TPartitionStats& stats) const {
514+
return stats.GetLocksBroken() >= OverloadedByTliPartitionBound;
452515
}
453516

454517
private:
@@ -458,7 +521,8 @@ class TPartitionStatsCollector : public TActorBootstrapped<TPartitionStatsCollec
458521
TPathId DomainKey;
459522
ui64 SysViewProcessorId = 0;
460523

461-
double OverloadedPartitionBound = 0.7;
524+
double OverloadedByCpuPartitionBound = 0.7;
525+
ui64 OverloadedByTliPartitionBound = 1;
462526
TDuration ProcessOverloadedInterval = TDuration::Seconds(15);
463527

464528
typedef ui32 TFollowerId;
@@ -473,22 +537,23 @@ class TPartitionStatsCollector : public TActorBootstrapped<TPartitionStatsCollec
473537
TString Path;
474538
};
475539

476-
struct TOverloadedFollower {
540+
struct TFollowerStats {
477541
TShardIdx ShardIdx;
478542
TFollowerId FollowerId;
479543

480-
bool operator<(const TOverloadedFollower &other) const {
544+
bool operator<(const TFollowerStats &other) const {
481545
return std::tie(ShardIdx, FollowerId) < std::tie(other.ShardIdx, other.FollowerId);
482546
}
483547

484-
bool operator==(const TOverloadedFollower &other) const {
548+
bool operator==(const TFollowerStats &other) const {
485549
return std::tie(ShardIdx, FollowerId) == std::tie(other.ShardIdx, other.FollowerId);
486550
}
487551
};
488552

489553
struct TDomainTables {
490554
std::map<TPathId, TTableStats> Stats;
491-
std::unordered_map<TPathId, std::set<TOverloadedFollower>> Overloaded;
555+
std::unordered_map<TPathId, std::set<TFollowerStats>> OverloadedByCpu;
556+
std::unordered_map<TPathId, std::set<TFollowerStats>> OverloadedByTli;
492557
};
493558
std::unordered_map<TPathId, TDomainTables> DomainTables;
494559

@@ -694,6 +759,15 @@ class TPartitionStatsScan : public TScanActorBase<TPartitionStatsScan> {
694759
insert({TSchema::FollowerId::ColumnId, [] (const TPartitionStatsResult&, const TPartitionStats&, const TPartitionStats& stats) {
695760
return TCell::Make<ui32>(stats.GetFollowerId());
696761
}});
762+
insert({TSchema::LocksAcquired::ColumnId, [] (const TPartitionStatsResult&, const TPartitionStats&, const TPartitionStats& stats) {
763+
return TCell::Make<ui64>(stats.GetLocksAcquired());
764+
}});
765+
insert({TSchema::LocksWholeShard::ColumnId, [] (const TPartitionStatsResult&, const TPartitionStats&, const TPartitionStats& stats) {
766+
return TCell::Make<ui64>(stats.GetLocksWholeShard());
767+
}});
768+
insert({TSchema::LocksBroken::ColumnId, [] (const TPartitionStatsResult&, const TPartitionStats&, const TPartitionStats& stats) {
769+
return TCell::Make<ui64>(stats.GetLocksBroken());
770+
}});
697771
}
698772
};
699773
static TExtractorsMap extractors;

0 commit comments

Comments
 (0)