Skip to content

Commit 58580f8

Browse files
swalrus1swalrus1
authored
add tx progress counters (#7407)
1 parent a8e55aa commit 58580f8

23 files changed

+316
-196
lines changed

‎ydb/core/tx/columnshard/columnshard.cpp

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -339,14 +339,9 @@ void TColumnShard::FillOlapStats(
339339
resourceMetrics->Fill(*ev->Record.MutableTabletMetrics());
340340
}
341341

342-
TTableStatsBuilder statsBuilder(*ev->Record.MutableTableStats());
343-
statsBuilder.FillColumnTableStats(*Counters.GetColumnTablesCounters());
344-
statsBuilder.FillTabletStats(*Counters.GetTabletCounters());
345-
statsBuilder.FillBackgroundControllerStats(*Counters.GetBackgroundControllerCounters());
346-
statsBuilder.FillScanCountersStats(Counters.GetScanCounters());
347-
statsBuilder.FillExecutorStats(*Executor());
348342
if (TablesManager.HasPrimaryIndex()) {
349-
statsBuilder.FillColumnEngineStats(TablesManager.MutablePrimaryIndex().GetTotalStats());
343+
TTableStatsBuilder statsBuilder(Counters, Executor(), TablesManager.MutablePrimaryIndex());
344+
statsBuilder.FillTotalTableStats(*ev->Record.MutableTableStats());
350345
}
351346
}
352347

@@ -355,6 +350,9 @@ void TColumnShard::FillColumnTableStats(
355350
std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev
356351
) {
357352
auto tables = TablesManager.GetTables();
353+
std::optional<TTableStatsBuilder> tableStatsBuilder =
354+
TablesManager.HasPrimaryIndex() ? std::make_optional<TTableStatsBuilder>(Counters, Executor(), TablesManager.MutablePrimaryIndex())
355+
: std::nullopt;
358356

359357
LOG_S_DEBUG("There are stats for " << tables.size() << " tables");
360358
for (const auto& [pathId, _] : tables) {
@@ -372,17 +370,8 @@ void TColumnShard::FillColumnTableStats(
372370
resourceMetrics->Fill(*periodicTableStats->MutableTabletMetrics());
373371
}
374372

375-
TTableStatsBuilder statsBuilder(*periodicTableStats->MutableTableStats());
376-
statsBuilder.FillColumnTableStats(*Counters.GetColumnTablesCounters()->GetPathIdCounter(pathId));
377-
statsBuilder.FillTabletStats(*Counters.GetTabletCounters());
378-
statsBuilder.FillBackgroundControllerStats(*Counters.GetBackgroundControllerCounters(), pathId);
379-
statsBuilder.FillScanCountersStats(Counters.GetScanCounters());
380-
statsBuilder.FillExecutorStats(*Executor());
381-
if (TablesManager.HasPrimaryIndex()) {
382-
auto columnEngineStats = TablesManager.GetPrimaryIndexSafe().GetStats().FindPtr(pathId);
383-
if (columnEngineStats && *columnEngineStats) {
384-
statsBuilder.FillColumnEngineStats(**columnEngineStats);
385-
}
373+
if (tableStatsBuilder) {
374+
tableStatsBuilder->FillTableStats(pathId, *(periodicTableStats->MutableTableStats()));
386375
}
387376

388377
LOG_S_TRACE("Add stats for table, tableLocalID=" << pathId);

‎ydb/core/tx/columnshard/columnshard__scan.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ void TColumnShard::Handle(TEvColumnShard::TEvScan::TPtr& ev, const TActorContext
3030
return;
3131
}
3232

33-
Counters.GetColumnTablesCounters()->GetPathIdCounter(record.GetLocalPathId())->OnAccess();
33+
Counters.GetColumnTablesCounters()->GetPathIdCounter(record.GetLocalPathId())->OnReadEvent();
3434
ScanTxInFlight.insert({txId, TAppData::TimeProvider->Now()});
3535
Counters.GetTabletCounters()->SetCounter(COUNTER_SCAN_IN_FLY, ScanTxInFlight.size());
3636
Execute(new NOlap::NReader::TTxScan(this, ev), ctx);

‎ydb/core/tx/columnshard/columnshard__write.cpp

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,22 @@ void TColumnShard::OverloadWriteFail(const EOverloadStatus overloadReason, const
1818
Counters.GetTabletCounters()->IncCounter(COUNTER_WRITE_FAIL);
1919
switch (overloadReason) {
2020
case EOverloadStatus::Disk:
21-
Counters.GetCSCounters().OnWriteOverloadDisk();
21+
Counters.OnWriteOverloadDisk();
2222
break;
2323
case EOverloadStatus::InsertTable:
24-
Counters.GetCSCounters().OnWriteOverloadInsertTable(writeData.GetSize());
24+
Counters.OnWriteOverloadInsertTable(writeData.GetSize());
2525
break;
2626
case EOverloadStatus::OverloadMetadata:
27-
Counters.GetCSCounters().OnWriteOverloadMetadata(writeData.GetSize());
27+
Counters.OnWriteOverloadMetadata(writeData.GetSize());
2828
break;
2929
case EOverloadStatus::ShardTxInFly:
30-
Counters.GetCSCounters().OnWriteOverloadShardTx(writeData.GetSize());
30+
Counters.OnWriteOverloadShardTx(writeData.GetSize());
3131
break;
3232
case EOverloadStatus::ShardWritesInFly:
33-
Counters.GetCSCounters().OnWriteOverloadShardWrites(writeData.GetSize());
33+
Counters.OnWriteOverloadShardWrites(writeData.GetSize());
3434
break;
3535
case EOverloadStatus::ShardWritesSizeInFly:
36-
Counters.GetCSCounters().OnWriteOverloadShardWritesSize(writeData.GetSize());
36+
Counters.OnWriteOverloadShardWritesSize(writeData.GetSize());
3737
break;
3838
case EOverloadStatus::None:
3939
Y_ABORT("invalid function usage");
@@ -86,7 +86,7 @@ void TColumnShard::Handle(TEvPrivate::TEvWriteBlobsResult::TPtr& ev, const TActo
8686
auto baseAggregations = wBuffer.GetAggregations();
8787
wBuffer.InitReplyReceived(TMonotonic::Now());
8888

89-
auto wg = Counters.GetWritesMonitor()->OnFinishWrite(wBuffer.GetSumSize(), wBuffer.GetAggregations().size());
89+
Counters.GetWritesMonitor()->OnFinishWrite(wBuffer.GetSumSize(), wBuffer.GetAggregations().size());
9090

9191
for (auto&& aggr : baseAggregations) {
9292
const auto& writeMeta = aggr->GetWriteMeta();
@@ -158,7 +158,7 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex
158158
const TString dedupId = record.GetDedupId();
159159
const auto source = ev->Sender;
160160

161-
Counters.GetColumnTablesCounters()->GetPathIdCounter(tableId)->OnUpdate();
161+
Counters.GetColumnTablesCounters()->GetPathIdCounter(tableId)->OnWriteEvent();
162162

163163
std::optional<ui32> granuleShardingVersion;
164164
if (record.HasGranuleShardingVersion()) {
@@ -397,7 +397,7 @@ void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActor
397397
return;
398398
}
399399

400-
auto wg = Counters.GetWritesMonitor()->OnStartWrite(arrowData->GetSize());
400+
Counters.GetWritesMonitor()->OnStartWrite(arrowData->GetSize());
401401

402402
std::optional<ui32> granuleShardingVersionId;
403403
if (record.HasGranuleShardingVersionId()) {

‎ydb/core/tx/columnshard/columnshard_impl.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,15 +65,15 @@ NTabletPipe::TClientConfig GetPipeClientConfig() {
6565
TColumnShard::TColumnShard(TTabletStorageInfo* info, const TActorId& tablet)
6666
: TActor(&TThis::StateInit)
6767
, TTabletExecutedFlat(info, tablet, nullptr)
68+
, TabletCountersHolder(new TProtobufTabletCounters<ESimpleCounters_descriptor, ECumulativeCounters_descriptor,
69+
EPercentileCounters_descriptor, ETxTypes_descriptor>())
70+
, Counters(*TabletCountersHolder)
6871
, ProgressTxController(std::make_unique<TTxController>(*this))
6972
, StoragesManager(std::make_shared<NOlap::TStoragesManager>(*this))
7073
, DataLocksManager(std::make_shared<NOlap::NDataLocks::TManager>())
7174
, PeriodicWakeupActivationPeriod(NYDBTest::TControllers::GetColumnShardController()->GetPeriodicWakeupActivationPeriod(
7275
TSettings::DefaultPeriodicWakeupActivationPeriod))
7376
, StatsReportInterval(NYDBTest::TControllers::GetColumnShardController()->GetStatsReportInterval(TSettings::DefaultStatsReportInterval))
74-
, TabletCountersHolder(new TProtobufTabletCounters<ESimpleCounters_descriptor, ECumulativeCounters_descriptor,
75-
EPercentileCounters_descriptor, ETxTypes_descriptor>())
76-
, Counters(*TabletCountersHolder)
7777
, InFlightReadsTracker(StoragesManager, Counters.GetRequestsTracingCounters())
7878
, TablesManager(StoragesManager, info->TabletID)
7979
, Subscribers(std::make_shared<NSubscriber::TManager>(*this))

‎ydb/core/tx/columnshard/columnshard_impl.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,9 @@ class TColumnShard
399399
}
400400

401401
private:
402+
std::unique_ptr<TTabletCountersBase> TabletCountersHolder;
403+
TCountersManager Counters;
404+
402405
std::unique_ptr<TTxController> ProgressTxController;
403406
std::unique_ptr<TOperationsManager> OperationsManager;
404407
std::shared_ptr<NOlap::NDataSharing::TSessionsManager> SharingSessionsManager;
@@ -445,9 +448,6 @@ class TColumnShard
445448
TActorId BufferizationWriteActorId;
446449
TActorId StatsReportPipe;
447450

448-
std::unique_ptr<TTabletCountersBase> TabletCountersHolder;
449-
TCountersManager Counters;
450-
451451
TInFlightReadsTracker InFlightReadsTracker;
452452
TTablesManager TablesManager;
453453
std::shared_ptr<NSubscriber::TManager> Subscribers;

‎ydb/core/tx/columnshard/counters/aggregation/table_stats.cpp

Lines changed: 0 additions & 41 deletions
This file was deleted.

‎ydb/core/tx/columnshard/counters/aggregation/table_stats.h

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,49 @@
11
#pragma once
22

3-
#include <ydb/core/tx/columnshard/counters/scan.h>
3+
#include <ydb/core/protos/table_stats.pb.h>
44
#include <ydb/core/tablet_flat/tablet_flat_executor.h>
5-
#include <ydb/core/tx/columnshard/counters/column_tables.h>
6-
#include <ydb/core/tx/columnshard/counters/tablet_counters.h>
7-
#include <ydb/core/tx/columnshard/counters/background_controller.h>
5+
#include <ydb/core/tx/columnshard/counters/counters_manager.h>
86
#include <ydb/core/tx/columnshard/engines/column_engine.h>
9-
#include <ydb/core/protos/table_stats.pb.h>
107

118
namespace NKikimr::NColumnShard {
129

1310
class TTableStatsBuilder {
1411
private:
15-
::NKikimrTableStats::TTableStats& TableStats;
12+
TCountersManager& Counters;
13+
const NTabletFlatExecutor::NFlatExecutorSetup::IExecutor& Executor;
14+
NOlap::IColumnEngine& ColumnEngine;
1615

1716
public:
18-
TTableStatsBuilder(::NKikimrTableStats::TTableStats& tableStats)
19-
: TableStats(tableStats) {
17+
TTableStatsBuilder(
18+
TCountersManager& counters, const NTabletFlatExecutor::NFlatExecutorSetup::IExecutor* executor, NOlap::IColumnEngine& columnEngine)
19+
: Counters(counters)
20+
, Executor(*executor)
21+
, ColumnEngine(columnEngine) {
2022
}
2123

22-
void FillColumnTableStats(const TSingleColumnTableCounters& stats);
23-
void FillColumnTableStats(const TColumnTablesCounters& stats);
24-
25-
void FillTabletStats(const TTabletCountersHandle& stats);
24+
void FillTableStats(ui64 pathId, ::NKikimrTableStats::TTableStats& tableStats) {
25+
Counters.FillTableStats(pathId, tableStats);
2626

27-
void FillBackgroundControllerStats(const TBackgroundControllerCounters& stats, ui64 pathId);
28-
void FillBackgroundControllerStats(const TBackgroundControllerCounters& stats);
27+
auto columnEngineStats = ColumnEngine.GetStats().FindPtr(pathId);
28+
if (columnEngineStats && *columnEngineStats) {
29+
auto activeStats = (*columnEngineStats)->Active();
30+
tableStats.SetRowCount(activeStats.Rows);
31+
tableStats.SetDataSize(activeStats.Bytes);
32+
tableStats.SetPartCount(activeStats.Portions);
33+
}
34+
}
2935

30-
void FillScanCountersStats(const TScanCounters& stats);
36+
void FillTotalTableStats(::NKikimrTableStats::TTableStats& tableStats) {
37+
Counters.FillTotalTableStats(tableStats);
3138

32-
void FillExecutorStats(const NTabletFlatExecutor::NFlatExecutorSetup::IExecutor& executor);
39+
tableStats.SetInFlightTxCount(Executor.GetStats().TxInFly);
40+
tableStats.SetHasLoanedParts(Executor.HasLoanedParts());
3341

34-
void FillColumnEngineStats(const NOlap::TColumnEngineStats& stats);
42+
auto activeStats = ColumnEngine.GetTotalStats().Active();
43+
tableStats.SetRowCount(activeStats.Rows);
44+
tableStats.SetDataSize(activeStats.Bytes);
45+
tableStats.SetPartCount(activeStats.Portions);
46+
}
3547
};
3648

37-
} // namespace NKikimr::NColumnShard
49+
} // namespace NKikimr::NColumnShard

‎ydb/core/tx/columnshard/counters/aggregation/ya.make

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
LIBRARY()
22

3-
SRCS(
4-
table_stats.cpp
5-
)
3+
SRCS()
64

75
PEERDIR(
86
ydb/core/protos

‎ydb/core/tx/columnshard/counters/column_tables.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,11 @@ class TSingleColumnTableCounters {
4747
, TotalLastUpdateTime(owner.LastUpdateTime) {
4848
}
4949

50-
void OnAccess() {
50+
void OnReadEvent() {
5151
UpdateLastAccessTime(TAppData::TimeProvider->Now());
5252
}
5353

54-
void OnUpdate() {
54+
void OnWriteEvent() {
5555
TInstant now = TAppData::TimeProvider->Now();
5656
UpdateLastUpdateTime(now);
5757
UpdateLastAccessTime(now);

‎ydb/core/tx/columnshard/counters/columnshard.cpp

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,10 @@
66

77
namespace NKikimr::NColumnShard {
88

9-
TCSCounters::TCSCounters(std::shared_ptr<const TTabletCountersHandle> tabletCounters)
9+
TCSCounters::TCSCounters()
1010
: TBase("CS")
11-
, TabletCounters(std::move(tabletCounters))
12-
, Initialization(*this) {
13-
Y_ABORT_UNLESS(TabletCounters);
14-
11+
, Initialization(*this)
12+
, TxProgress(*this) {
1513
StartBackgroundCount = TBase::GetDeriviative("StartBackground/Count");
1614
TooEarlyBackgroundCount = TBase::GetDeriviative("TooEarlyBackground/Count");
1715
SetupCompactionCount = TBase::GetDeriviative("SetupCompaction/Count");

0 commit comments

Comments
 (0)