Skip to content

Commit bb1c803

Browse files
ivanmorozov333ivanmorozov333
authored
correct portions index usage for control memory scan intervals (#8135)
1 parent 60183c7 commit bb1c803

File tree

4 files changed

+54
-44
lines changed

4 files changed

+54
-44
lines changed

‎ydb/core/tx/columnshard/engines/column_engine_logs.cpp

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -510,20 +510,18 @@ std::shared_ptr<TSelectInfo> TColumnEngineForLogs::Select(ui64 pathId, TSnapshot
510510
return out;
511511
}
512512

513-
for (const auto& [indexKey, keyPortions] : spg->GetPortionsIndex().GetPoints()) {
514-
for (auto&& [_, portionInfo] : keyPortions.GetStart()) {
515-
if (!portionInfo->IsVisible(snapshot)) {
516-
continue;
517-
}
518-
Y_ABORT_UNLESS(portionInfo->Produced());
519-
const bool skipPortion = !pkRangesFilter.IsPortionInUsage(*portionInfo);
520-
AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", skipPortion ? "portion_skipped" : "portion_selected")
521-
("pathId", pathId)("portion", portionInfo->DebugString());
522-
if (skipPortion) {
523-
continue;
524-
}
525-
out->PortionsOrderedPK.emplace_back(portionInfo);
513+
for (const auto& [_, portionInfo] : spg->GetPortions()) {
514+
if (!portionInfo->IsVisible(snapshot)) {
515+
continue;
516+
}
517+
Y_ABORT_UNLESS(portionInfo->Produced());
518+
const bool skipPortion = !pkRangesFilter.IsPortionInUsage(*portionInfo);
519+
AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", skipPortion ? "portion_skipped" : "portion_selected")("pathId", pathId)(
520+
"portion", portionInfo->DebugString());
521+
if (skipPortion) {
522+
continue;
526523
}
524+
out->PortionsOrderedPK.emplace_back(portionInfo);
527525
}
528526

529527
return out;

‎ydb/core/tx/columnshard/engines/storage/granule/granule.cpp

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,9 @@ bool TGranuleMeta::ErasePortion(const ui64 portion) {
4747

4848
void TGranuleMeta::OnAfterChangePortion(const std::shared_ptr<TPortionInfo> portionAfter, NStorageOptimizer::IOptimizerPlanner::TModificationGuard* modificationGuard) {
4949
if (portionAfter) {
50-
PortionsIndex.AddPortion(portionAfter);
51-
5250
PortionInfoGuard.OnNewPortion(portionAfter);
5351
if (!portionAfter->HasRemoveSnapshot()) {
52+
PortionsIndex.AddPortion(portionAfter);
5453
if (modificationGuard) {
5554
modificationGuard->AddPortion(portionAfter);
5655
} else {
@@ -74,10 +73,9 @@ void TGranuleMeta::OnAfterChangePortion(const std::shared_ptr<TPortionInfo> port
7473

7574
void TGranuleMeta::OnBeforeChangePortion(const std::shared_ptr<TPortionInfo> portionBefore) {
7675
if (portionBefore) {
77-
PortionsIndex.RemovePortion(portionBefore);
78-
7976
PortionInfoGuard.OnDropPortion(portionBefore);
8077
if (!portionBefore->HasRemoveSnapshot()) {
78+
PortionsIndex.RemovePortion(portionBefore);
8179
OptimizerPlanner->StartModificationGuard().RemovePortion(portionBefore);
8280
ActualizationIndex->RemovePortion(portionBefore);
8381
}

‎ydb/core/tx/columnshard/engines/storage/granule/portions_index.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,11 @@ void TPortionsIndex::RemovePortion(const std::shared_ptr<TPortionInfo>& p) {
5555
auto itTo = Points.find(p->IndexKeyEnd());
5656
AFL_VERIFY(itTo != Points.end());
5757
{
58-
const TPortionInfoStat stat(p->GetMinMemoryForReadColumns({}), p->GetTotalBlobBytes());
58+
const TPortionInfoStat stat(p);
5959
auto it = itFrom;
6060
while (true) {
6161
RemoveFromMemoryUsageControl(it->second.GetIntervalStats());
62-
it->second.RemoveContained(p->GetPortionId(), stat);
62+
it->second.RemoveContained(stat);
6363
RawMemoryUsage.Add(it->second.GetIntervalStats().GetMinRawBytes());
6464
BlobMemoryUsage.Add(it->second.GetIntervalStats().GetBlobBytes());
6565
if (it == itTo) {
@@ -98,10 +98,10 @@ void TPortionsIndex::AddPortion(const std::shared_ptr<TPortionInfo>& p) {
9898
itTo->second.AddFinish(p);
9999

100100
auto it = itFrom;
101-
const TPortionInfoStat stat(p->GetMinMemoryForReadColumns({}), p->GetTotalBlobBytes());
101+
const TPortionInfoStat stat(p);
102102
while (true) {
103103
RemoveFromMemoryUsageControl(it->second.GetIntervalStats());
104-
it->second.AddContained(p->GetPortionId(), stat);
104+
it->second.AddContained(stat);
105105
RawMemoryUsage.Add(it->second.GetIntervalStats().GetMinRawBytes());
106106
BlobMemoryUsage.Add(it->second.GetIntervalStats().GetBlobBytes());
107107
if (it == itTo) {

‎ydb/core/tx/columnshard/engines/storage/granule/portions_index.h

Lines changed: 37 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,29 +10,41 @@ namespace NKikimr::NOlap::NGranule::NPortionsIndex {
1010

1111
class TPortionInfoStat {
1212
private:
13+
std::shared_ptr<TPortionInfo> PortionInfo;
1314
YDB_READONLY(ui64, MinRawBytes, 0);
1415
YDB_READONLY(ui64, BlobBytes, 0);
1516

1617
public:
17-
TPortionInfoStat() = default;
18-
19-
TPortionInfoStat(const ui64 rawBytes, const ui64 blobBytes)
20-
: MinRawBytes(rawBytes)
21-
, BlobBytes(blobBytes)
18+
TPortionInfoStat(const std::shared_ptr<TPortionInfo>& portionInfo)
19+
: PortionInfo(portionInfo)
20+
, MinRawBytes(PortionInfo->GetMinMemoryForReadColumns({}))
21+
, BlobBytes(PortionInfo->GetTotalBlobBytes())
2222
{
2323

2424
}
2525

26+
const TPortionInfo& GetPortionInfoVerified() const {
27+
AFL_VERIFY(PortionInfo);
28+
return *PortionInfo;
29+
}
30+
};
31+
32+
class TIntervalInfoStat {
33+
private:
34+
YDB_READONLY(ui64, MinRawBytes, 0);
35+
YDB_READONLY(ui64, BlobBytes, 0);
36+
37+
public:
2638
void Add(const TPortionInfoStat& source) {
27-
MinRawBytes += source.MinRawBytes;
28-
BlobBytes += source.BlobBytes;
39+
MinRawBytes += source.GetMinRawBytes();
40+
BlobBytes += source.GetBlobBytes();
2941
}
3042

3143
void Sub(const TPortionInfoStat& source) {
32-
AFL_VERIFY(MinRawBytes >= source.MinRawBytes);
33-
MinRawBytes -= source.MinRawBytes;
34-
AFL_VERIFY(BlobBytes >= source.BlobBytes);
35-
BlobBytes -= source.BlobBytes;
44+
AFL_VERIFY(MinRawBytes >= source.GetMinRawBytes());
45+
MinRawBytes -= source.GetMinRawBytes();
46+
AFL_VERIFY(BlobBytes >= source.GetBlobBytes());
47+
BlobBytes -= source.GetBlobBytes();
3648
AFL_VERIFY(!!BlobBytes == !!MinRawBytes);
3749
}
3850

@@ -46,20 +58,20 @@ class TPortionsPKPoint {
4658
THashMap<ui64, std::shared_ptr<TPortionInfo>> Start;
4759
THashMap<ui64, std::shared_ptr<TPortionInfo>> Finish;
4860
THashMap<ui64, TPortionInfoStat> PortionIds;
49-
YDB_READONLY_DEF(TPortionInfoStat, IntervalStats);
61+
YDB_READONLY_DEF(TIntervalInfoStat, IntervalStats);
5062

5163
public:
5264
const THashMap<ui64, std::shared_ptr<TPortionInfo>>& GetStart() const {
5365
return Start;
5466
}
5567

5668
void ProvidePortions(const TPortionsPKPoint& source) {
57-
IntervalStats = TPortionInfoStat();
69+
IntervalStats = TIntervalInfoStat();
5870
for (auto&& [i, stat] : source.PortionIds) {
5971
if (source.Finish.contains(i)) {
6072
continue;
6173
}
62-
AddContained(i, stat);
74+
AddContained(stat);
6375
}
6476
}
6577

@@ -71,17 +83,19 @@ class TPortionsPKPoint {
7183
return Start.empty() && Finish.empty();
7284
}
7385

74-
void AddContained(const ui32 portionId, const TPortionInfoStat& stat) {
75-
IntervalStats.Add(stat);
76-
AFL_VERIFY(PortionIds.emplace(portionId, stat).second);
86+
void AddContained(const TPortionInfoStat& stat) {
87+
if (!stat.GetPortionInfoVerified().HasRemoveSnapshot()) {
88+
IntervalStats.Add(stat);
89+
}
90+
AFL_VERIFY(PortionIds.emplace(stat.GetPortionInfoVerified().GetPortionId(), stat).second);
7791
}
7892

79-
void RemoveContained(const ui32 portionId, const TPortionInfoStat& stat) {
80-
IntervalStats.Sub(stat);
81-
AFL_VERIFY(PortionIds.erase(portionId));
82-
if (PortionIds.empty()) {
83-
AFL_VERIFY(!IntervalStats);
93+
void RemoveContained(const TPortionInfoStat& stat) {
94+
if (!stat.GetPortionInfoVerified().HasRemoveSnapshot()) {
95+
IntervalStats.Sub(stat);
8496
}
97+
AFL_VERIFY(PortionIds.erase(stat.GetPortionInfoVerified().GetPortionId()));
98+
AFL_VERIFY(PortionIds.size() || !IntervalStats);
8599
}
86100

87101
void RemoveStart(const std::shared_ptr<TPortionInfo>& p) {
@@ -162,7 +176,7 @@ class TPortionsIndex {
162176
return it;
163177
}
164178

165-
void RemoveFromMemoryUsageControl(const TPortionInfoStat& stat) {
179+
void RemoveFromMemoryUsageControl(const TIntervalInfoStat& stat) {
166180
RawMemoryUsage.Remove(stat.GetMinRawBytes());
167181
BlobMemoryUsage.Remove(stat.GetBlobBytes());
168182
}

0 commit comments

Comments
 (0)