Skip to content

Commit b17d771

Browse files
committed
Bill vector index build CPU (#20168)
1 parent 4f9dad1 commit b17d771

File tree

16 files changed

+168
-59
lines changed

16 files changed

+168
-59
lines changed

ydb/core/protos/index_builder.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ message TMeteringStats {
3333
optional uint64 UploadBytes = 2;
3434
optional uint64 ReadRows = 3;
3535
optional uint64 ReadBytes = 4;
36+
optional uint64 CpuTimeUs = 5;
3637
}
3738

3839
message TIndexBuildSettings {

ydb/core/tablet_flat/flat_scan_actor.h

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,7 @@ namespace NOps {
438438
void SendStat(const TStatState& stat)
439439
{
440440
ui64 elapsedUs = 1000000. * NHPTimer::GetSeconds(stat.ElapsedCycles());
441-
441+
TotalCpuTimeUs += elapsedUs;
442442
SendToOwner(new TEvScanStat(elapsedUs, stat.Seen, stat.Skipped));
443443
}
444444

@@ -474,9 +474,9 @@ namespace NOps {
474474
processed += stat.UpdateRows(Seen, Skipped);
475475

476476
if (ready == NTable::EReady::Gone) {
477-
Terminate(EAbort::None);
478477
stat.UpdateCycles();
479478
SendStat(stat);
479+
Terminate(EAbort::None);
480480
return;
481481
}
482482

@@ -493,9 +493,9 @@ namespace NOps {
493493
if (!MayProgress()) {
494494
// We must honor EReady::Gone from an implicit callback
495495
if (ImplicitPageFault() == NTable::EReady::Gone) {
496-
Terminate(EAbort::None);
497496
stat.UpdateCycles();
498497
SendStat(stat);
498+
Terminate(EAbort::None);
499499
return;
500500
}
501501

@@ -695,6 +695,11 @@ namespace NOps {
695695
Send(Owner, event.Release(), flags);
696696
}
697697

698+
ui64 GetTotalCpuTimeUs() const override
699+
{
700+
return TotalCpuTimeUs;
701+
}
702+
698703
private:
699704
struct TBlobQueueRequest {
700705
TActorId Sender;
@@ -724,6 +729,7 @@ namespace NOps {
724729

725730
const NHPTimer::STime MaxCyclesPerIteration;
726731
static constexpr ui64 MinRowsPerCheck = 1000;
732+
ui64 TotalCpuTimeUs = 0;
727733
};
728734

729735
}

ydb/core/tablet_flat/flat_scan_iface.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ namespace NTable {
7575
class IDriver {
7676
public:
7777
virtual void Touch(EScan) noexcept = 0;
78+
79+
virtual ui64 GetTotalCpuTimeUs() const = 0;
7880
};
7981

8082

ydb/core/tx/datashard/build_index/local_kmeans.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ class TLocalKMeansScan: public TActor<TLocalKMeansScan>, public NTable::IScan {
166166
auto& record = Response->Record;
167167
record.MutableMeteringStats()->SetReadRows(ReadRows);
168168
record.MutableMeteringStats()->SetReadBytes(ReadBytes);
169+
record.MutableMeteringStats()->SetCpuTimeUs(Driver->GetTotalCpuTimeUs());
169170

170171
Uploader.Finish(record, abort);
171172

ydb/core/tx/datashard/build_index/prefix_kmeans.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ class TPrefixKMeansScan: public TActor<TPrefixKMeansScan>, public NTable::IScan
199199
auto& record = Response->Record;
200200
record.MutableMeteringStats()->SetReadRows(ReadRows);
201201
record.MutableMeteringStats()->SetReadBytes(ReadBytes);
202+
record.MutableMeteringStats()->SetCpuTimeUs(Driver->GetTotalCpuTimeUs());
202203

203204
Uploader.Finish(record, abort);
204205

ydb/core/tx/datashard/build_index/recompute_kmeans.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ class TRecomputeKMeansScan: public TActor<TRecomputeKMeansScan>, public NTable::
102102
auto& record = Response->Record;
103103
record.MutableMeteringStats()->SetReadRows(ReadRows);
104104
record.MutableMeteringStats()->SetReadBytes(ReadBytes);
105+
record.MutableMeteringStats()->SetCpuTimeUs(Driver->GetTotalCpuTimeUs());
105106

106107
if (abort != EAbort::None) {
107108
record.SetStatus(NKikimrIndexBuilder::EBuildStatus::ABORTED);

ydb/core/tx/datashard/build_index/reshuffle_kmeans.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ class TReshuffleKMeansScan: public TActor<TReshuffleKMeansScan>, public NTable::
138138
auto& record = Response->Record;
139139
record.MutableMeteringStats()->SetReadRows(ReadRows);
140140
record.MutableMeteringStats()->SetReadBytes(ReadBytes);
141+
record.MutableMeteringStats()->SetCpuTimeUs(Driver->GetTotalCpuTimeUs());
141142

142143
Uploader.Finish(record, abort);
143144

ydb/core/tx/datashard/build_index/sample_k.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ class TSampleKScan final: public TActor<TSampleKScan>, public NTable::IScan {
171171
auto& record = Response->Record;
172172
record.MutableMeteringStats()->SetReadRows(ReadRows);
173173
record.MutableMeteringStats()->SetReadBytes(ReadBytes);
174+
record.MutableMeteringStats()->SetCpuTimeUs(Driver->GetTotalCpuTimeUs());
174175

175176
if (HasBuildError) {
176177
record.SetStatus(NKikimrIndexBuilder::EBuildStatus::BUILD_ERROR);

ydb/core/tx/datashard/datashard_ut_incremental_restore_scan.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,13 @@ class TDriverMock
2121
public:
2222
std::optional<NTable::EScan> LastScan;
2323

24-
void Touch(NTable::EScan scan) noexcept {
24+
void Touch(NTable::EScan scan) noexcept override {
2525
LastScan = scan;
2626
}
27+
28+
ui64 GetTotalCpuTimeUs() const override {
29+
return 0;
30+
}
2731
};
2832

2933
class TCbExecutorActor : public TActorBootstrapped<TCbExecutorActor> {

ydb/core/tx/schemeshard/schemeshard_billing_helpers.cpp

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ TMeteringStats& operator += (TMeteringStats& value, const TMeteringStats& other)
2323
value.SetUploadBytes(value.GetUploadBytes() + other.GetUploadBytes());
2424
value.SetReadRows(value.GetReadRows() + other.GetReadRows());
2525
value.SetReadBytes(value.GetReadBytes() + other.GetReadBytes());
26+
value.SetCpuTimeUs(value.GetCpuTimeUs() + other.GetCpuTimeUs());
2627
return value;
2728
}
2829

@@ -39,6 +40,7 @@ TMeteringStats& operator -= (TMeteringStats& value, const TMeteringStats& other)
3940
value.SetUploadBytes(safeSub(value.GetUploadBytes(), other.GetUploadBytes()));
4041
value.SetReadRows(safeSub(value.GetReadRows(), other.GetReadRows()));
4142
value.SetReadBytes(safeSub(value.GetReadBytes(), other.GetReadBytes()));
43+
value.SetCpuTimeUs(safeSub(value.GetCpuTimeUs(), other.GetCpuTimeUs()));
4244
return value;
4345
}
4446

@@ -57,14 +59,16 @@ TMeteringStats TMeteringStatsHelper::ZeroValue() {
5759
value.SetUploadBytes(0);
5860
value.SetReadRows(0);
5961
value.SetReadBytes(0);
62+
value.SetCpuTimeUs(0);
6063
return value;
6164
}
6265

6366
bool TMeteringStatsHelper::IsZero(TMeteringStats& value) {
6467
return value.GetUploadRows() == 0
6568
&& value.GetUploadBytes() == 0
6669
&& value.GetReadRows() == 0
67-
&& value.GetReadBytes() == 0;
70+
&& value.GetReadBytes() == 0
71+
&& value.GetCpuTimeUs() == 0;
6872
}
6973

7074
ui64 TRUCalculator::ReadTable(ui64 bytes) {
@@ -84,15 +88,28 @@ ui64 TRUCalculator::BulkUpsert(ui64 bytes, ui64 rows) {
8488
return (Max(rows, (bytes + 1_KB - 1) / 1_KB) + 1) / 2;
8589
}
8690

91+
ui64 TRUCalculator::CPU(ui64 сpuTimeUs) {
92+
// The sum is divided by the CPU time increment of 1.5 ms, rounded down and converted to RU.
93+
// https://yandex.cloud/en-ru/docs/ydb/pricing/ru-yql
94+
return сpuTimeUs / 1500;
95+
}
96+
8797
ui64 TRUCalculator::Calculate(const TMeteringStats& stats, TString& explain) {
8898
// The cost of building an index is the sum of the cost of ReadTable from the source table and BulkUpsert to the index table.
8999
// https://yandex.cloud/en-ru/docs/ydb/pricing/ru-special#secondary-index
100+
101+
// To evaluate the YDB API request cost, the CPU cost and the I/O cost are calculated. A maximum from the calculated values is selected.
102+
// https://yandex.cloud/en-ru/docs/ydb/pricing/ru-yql
103+
90104
ui64 readTable = TRUCalculator::ReadTable(stats.GetReadBytes());
91105
ui64 bulkUpsert = TRUCalculator::BulkUpsert(stats.GetUploadBytes(), stats.GetUploadRows());
106+
ui64 cpu = TRUCalculator::CPU(stats.GetCpuTimeUs());
92107
explain = TStringBuilder()
93108
<< "ReadTable: " << readTable
94-
<< ", BulkUpsert: " << bulkUpsert;
95-
return readTable + bulkUpsert;
109+
<< ", BulkUpsert: " << bulkUpsert
110+
<< ", CPU: " << cpu;
111+
112+
return Max(readTable + bulkUpsert, cpu);
96113
}
97114

98115
}

0 commit comments

Comments
 (0)