Skip to content

Commit 73ea386

Browse files
zverevgenyblinkov
zverevgeny
authored and
blinkov
committed
minor improvements to cs scan (#14376)
1 parent a1c85a5 commit 73ea386

File tree

20 files changed

+46
-51
lines changed

20 files changed

+46
-51
lines changed

‎ydb/core/tx/columnshard/engines/column_engine_logs.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -443,7 +443,7 @@ std::shared_ptr<TSelectInfo> TColumnEngineForLogs::Select(
443443
} else if (!portionInfo->HasCommitSnapshot()) {
444444
continue;
445445
}
446-
const bool skipPortion = !pkRangesFilter.IsPortionInUsage(*portionInfo);
446+
const bool skipPortion = !pkRangesFilter.IsUsed(*portionInfo);
447447
AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", skipPortion ? "portion_skipped" : "portion_selected")("pathId", pathId)(
448448
"portion", portionInfo->DebugString());
449449
if (skipPortion) {
@@ -455,7 +455,7 @@ std::shared_ptr<TSelectInfo> TColumnEngineForLogs::Select(
455455
if (!portionInfo->IsVisible(snapshot, !withUncommitted)) {
456456
continue;
457457
}
458-
const bool skipPortion = !pkRangesFilter.IsPortionInUsage(*portionInfo);
458+
const bool skipPortion = !pkRangesFilter.IsUsed(*portionInfo);
459459
AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", skipPortion ? "portion_skipped" : "portion_selected")("pathId", pathId)(
460460
"portion", portionInfo->DebugString());
461461
if (skipPortion) {

‎ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ std::vector<TCommittedBlob> TInsertTable::Read(ui64 pathId, const std::optional<
153153
if (lockId || data.GetSnapshot() <= reqSnapshot) {
154154
auto start = data.GetMeta().GetFirstPK(pkSchema);
155155
auto finish = data.GetMeta().GetLastPK(pkSchema);
156-
if (pkRangesFilter && pkRangesFilter->IsPortionInPartialUsage(start, finish) == TPKRangeFilter::EUsageClass::DontUsage) {
156+
if (pkRangesFilter && !pkRangesFilter->IsUsed(start, finish)) {
157157
continue;
158158
}
159159
result.emplace_back(TCommittedBlob(data.GetBlobRange(), data.GetSnapshot(), data.GetInsertWriteId(), data.GetSchemaVersion(), data.GetMeta().GetRecordsCount(),
@@ -167,7 +167,7 @@ std::vector<TCommittedBlob> TInsertTable::Read(ui64 pathId, const std::optional<
167167
}
168168
auto start = data.GetMeta().GetFirstPK(pkSchema);
169169
auto finish = data.GetMeta().GetLastPK(pkSchema);
170-
if (pkRangesFilter && pkRangesFilter->IsPortionInPartialUsage(start, finish) == TPKRangeFilter::EUsageClass::DontUsage) {
170+
if (pkRangesFilter && !pkRangesFilter->IsUsed(start, finish)) {
171171
continue;
172172
}
173173
result.emplace_back(TCommittedBlob(data.GetBlobRange(), writeId, data.GetSchemaVersion(), data.GetMeta().GetRecordsCount(), start, finish,

‎ydb/core/tx/columnshard/engines/predicate/container.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ bool TPredicateContainer::IsInclude() const {
9595
return CompareType == NArrow::ECompareType::GREATER_OR_EQUAL || CompareType == NArrow::ECompareType::LESS_OR_EQUAL;
9696
}
9797

98-
bool TPredicateContainer::CrossRanges(const TPredicateContainer& ext) {
98+
bool TPredicateContainer::CrossRanges(const TPredicateContainer& ext) const {
9999
if (Object && ext.Object) {
100100
if (IsForwardInterval() == ext.IsForwardInterval()) {
101101
return true;

‎ydb/core/tx/columnshard/engines/predicate/container.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ class TPredicateContainer {
9898

9999
bool IsInclude() const;
100100

101-
bool CrossRanges(const TPredicateContainer& ext);
101+
bool CrossRanges(const TPredicateContainer& ext) const;
102102

103103
static TPredicateContainer BuildNullPredicateFrom() {
104104
return TPredicateContainer(NArrow::ECompareType::GREATER_OR_EQUAL);

‎ydb/core/tx/columnshard/engines/predicate/filter.cpp

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -85,15 +85,6 @@ std::set<ui32> TPKRangesFilter::GetColumnIds(const TIndexInfo& indexInfo) const
8585
return result;
8686
}
8787

88-
bool TPKRangesFilter::IsPortionInUsage(const TPortionInfo& info) const {
89-
for (auto&& i : SortedRanges) {
90-
if (i.IsPortionInUsage(info)) {
91-
return true;
92-
}
93-
}
94-
return SortedRanges.empty();
95-
}
96-
9788
bool TPKRangesFilter::CheckPoint(const NArrow::TReplaceKey& point) const {
9889
for (auto&& i : SortedRanges) {
9990
if (i.CheckPoint(point)) {
@@ -103,18 +94,21 @@ bool TPKRangesFilter::CheckPoint(const NArrow::TReplaceKey& point) const {
10394
return SortedRanges.empty();
10495
}
10596

106-
TPKRangeFilter::EUsageClass TPKRangesFilter::IsPortionInPartialUsage(const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& end) const {
97+
TPKRangeFilter::EUsageClass TPKRangesFilter::GetUsageClass(const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& end) const {
98+
if (SortedRanges.empty()) {
99+
return TPKRangeFilter::EUsageClass::FullUsage;
100+
}
107101
for (auto&& i : SortedRanges) {
108-
switch (i.IsPortionInPartialUsage(start, end)) {
102+
switch (i.GetUsageClass(start, end)) {
109103
case TPKRangeFilter::EUsageClass::FullUsage:
110104
return TPKRangeFilter::EUsageClass::FullUsage;
111105
case TPKRangeFilter::EUsageClass::PartialUsage:
112106
return TPKRangeFilter::EUsageClass::PartialUsage;
113-
case TPKRangeFilter::EUsageClass::DontUsage:
107+
case TPKRangeFilter::EUsageClass::NoUsage:
114108
break;
115109
}
116110
}
117-
return TPKRangeFilter::EUsageClass::DontUsage;
111+
return TPKRangeFilter::EUsageClass::NoUsage;
118112
}
119113

120114
TPKRangesFilter::TPKRangesFilter(const bool reverse)

‎ydb/core/tx/columnshard/engines/predicate/filter.h

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,14 @@ class TPKRangesFilter {
5959
return SortedRanges.end();
6060
}
6161

62-
bool IsPortionInUsage(const TPortionInfo& info) const;
63-
TPKRangeFilter::EUsageClass IsPortionInPartialUsage(const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& end) const;
62+
bool IsUsed(const TPortionInfo& info) const {
63+
return IsUsed(info.IndexKeyStart(), info.IndexKeyEnd());
64+
}
65+
66+
bool IsUsed(const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& end) const {
67+
return GetUsageClass(start, end) != TPKRangeFilter::EUsageClass::NoUsage;
68+
}
69+
TPKRangeFilter::EUsageClass GetUsageClass(const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& end) const;
6470
bool CheckPoint(const NArrow::TReplaceKey& point) const;
6571

6672
NArrow::TColumnFilter BuildFilter(const arrow::Datum& data) const;
@@ -90,11 +96,9 @@ class TPKRangesFilter {
9096
static TConclusion<TPKRangesFilter> BuildFromProto(const TProto& proto, const bool reverse, const std::vector<TNameTypeInfo>& ydbPk) {
9197
TPKRangesFilter result(reverse);
9298
for (auto& protoRange : proto.GetRanges()) {
93-
TSerializedTableRange range(protoRange);
9499
auto fromPredicate = std::make_shared<TPredicate>();
95100
auto toPredicate = std::make_shared<TPredicate>();
96-
TSerializedTableRange serializedRange(protoRange);
97-
std::tie(*fromPredicate, *toPredicate) = TPredicate::DeserializePredicatesRange(serializedRange, ydbPk);
101+
std::tie(*fromPredicate, *toPredicate) = TPredicate::DeserializePredicatesRange(TSerializedTableRange{protoRange}, ydbPk);
98102
auto status = result.Add(fromPredicate, toPredicate, NArrow::TStatusValidator::GetValid(NArrow::MakeArrowSchema(ydbPk)));
99103
if (status.IsFail()) {
100104
return status;

‎ydb/core/tx/columnshard/engines/predicate/range.cpp

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,11 @@ NKikimr::NArrow::TColumnFilter TPKRangeFilter::BuildFilter(const arrow::Datum& d
3939
return result.And(PredicateFrom.BuildFilter(data));
4040
}
4141

42-
bool TPKRangeFilter::IsPortionInUsage(const TPortionInfo& info) const {
43-
return IsPortionInPartialUsage(info.IndexKeyStart(), info.IndexKeyEnd()) != TPKRangeFilter::EUsageClass::DontUsage;
42+
bool TPKRangeFilter::IsUsed(const TPortionInfo& info) const {
43+
return GetUsageClass(info.IndexKeyStart(), info.IndexKeyEnd()) != TPKRangeFilter::EUsageClass::NoUsage;
4444
}
4545

46-
TPKRangeFilter::EUsageClass TPKRangeFilter::IsPortionInPartialUsage(const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& end) const {
46+
TPKRangeFilter::EUsageClass TPKRangeFilter::GetUsageClass(const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& end) const {
4747
{
4848
std::partial_ordering equalityStartWithFrom = std::partial_ordering::greater;
4949
if (const auto& from = PredicateFrom.GetReplaceKey()) {
@@ -66,25 +66,25 @@ TPKRangeFilter::EUsageClass TPKRangeFilter::IsPortionInPartialUsage(const NArrow
6666
if (const auto& from = PredicateFrom.GetReplaceKey()) {
6767
const std::partial_ordering equalityEndWithFrom = end.ComparePartNotNull(*from, from->Size());
6868
if (equalityEndWithFrom == std::partial_ordering::less) {
69-
return EUsageClass::DontUsage;
69+
return EUsageClass::NoUsage;
7070
} else if (equalityEndWithFrom == std::partial_ordering::equivalent) {
7171
if (PredicateFrom.IsInclude()) {
7272
return EUsageClass::PartialUsage;
7373
} else {
74-
return EUsageClass::DontUsage;
74+
return EUsageClass::NoUsage;
7575
}
7676
}
7777
}
7878

7979
if (const auto& to = PredicateTo.GetReplaceKey()) {
8080
const std::partial_ordering equalityStartWithTo = start.ComparePartNotNull(*to, to->Size());
8181
if (equalityStartWithTo == std::partial_ordering::greater) {
82-
return EUsageClass::DontUsage;
82+
return EUsageClass::NoUsage;
8383
} else if (equalityStartWithTo == std::partial_ordering::equivalent) {
8484
if (PredicateTo.IsInclude()) {
8585
return EUsageClass::PartialUsage;
8686
} else {
87-
return EUsageClass::DontUsage;
87+
return EUsageClass::NoUsage;
8888
}
8989
}
9090
}

‎ydb/core/tx/columnshard/engines/predicate/range.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,16 +38,16 @@ class TPKRangeFilter {
3838

3939
NArrow::TColumnFilter BuildFilter(const arrow::Datum& data) const;
4040

41-
bool IsPortionInUsage(const TPortionInfo& info) const;
41+
bool IsUsed(const TPortionInfo& info) const;
4242
bool CheckPoint(const NArrow::TReplaceKey& point) const;
4343

4444
enum class EUsageClass {
45-
DontUsage,
45+
NoUsage,
4646
PartialUsage,
4747
FullUsage
4848
};
4949

50-
EUsageClass IsPortionInPartialUsage(const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& end) const;
50+
EUsageClass GetUsageClass(const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& end) const;
5151

5252
std::set<ui32> GetColumnIds(const TIndexInfo& indexInfo) const;
5353
TString DebugString() const;

‎ydb/core/tx/columnshard/engines/reader/abstract/constructor.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ NKikimr::TConclusion<std::shared_ptr<TReadMetadataBase>> IScannerConstructor::Bu
7979
} else {
8080
(*result)->SetRequestedLimit(ItemsLimit);
8181
(*result)->SetScanIdentifier(read.GetScanIdentifier());
82-
return result.DetachResult();
82+
return result;
8383
}
8484
}
8585

‎ydb/core/tx/columnshard/engines/reader/common_reader/iterator/iterator.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@
44

55
namespace NKikimr::NOlap::NReader::NCommon {
66

7-
TColumnShardScanIterator::TColumnShardScanIterator(const std::shared_ptr<TReadContext>& context, const TReadMetadata::TConstPtr& readMetadata)
7+
TColumnShardScanIterator::TColumnShardScanIterator(const std::shared_ptr<TReadContext>& context)
88
: Context(context)
9-
, ReadMetadata(readMetadata)
9+
, ReadMetadata(context->GetReadMetadataPtrVerifiedAs<TReadMetadata>())
1010
, ReadyResults(context->GetCounters()) {
11-
IndexedData = readMetadata->BuildReader(Context);
11+
IndexedData = ReadMetadata->BuildReader(Context);
1212
Y_ABORT_UNLESS(Context->GetReadMetadata()->IsSorted());
1313
}
1414

‎ydb/core/tx/columnshard/engines/reader/common_reader/iterator/iterator.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ class TColumnShardScanIterator: public TScanIteratorBase {
6666
std::shared_ptr<IDataReader> IndexedData;
6767

6868
public:
69-
TColumnShardScanIterator(const std::shared_ptr<TReadContext>& context, const std::shared_ptr<const TReadMetadata>& readMetadata);
69+
TColumnShardScanIterator(const std::shared_ptr<TReadContext>& context);
7070
~TColumnShardScanIterator();
7171

7272
virtual TConclusionStatus Start() override {

‎ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
namespace NKikimr::NOlap::NReader::NPlain {
88

99
std::unique_ptr<TScanIteratorBase> TReadMetadata::StartScan(const std::shared_ptr<TReadContext>& readContext) const {
10-
return std::make_unique<TColumnShardScanIterator>(readContext, readContext->GetReadMetadataPtrVerifiedAs<TReadMetadata>());
10+
return std::make_unique<TColumnShardScanIterator>(readContext);
1111
}
1212

1313
TConclusionStatus TReadMetadata::DoInitCustom(

‎ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::DoGetColumnsFetchingPlan(c
4141
switch (source->GetUsageClass()) {
4242
case TPKRangeFilter::EUsageClass::PartialUsage:
4343
return true;
44-
case TPKRangeFilter::EUsageClass::DontUsage:
44+
case TPKRangeFilter::EUsageClass::NoUsage:
4545
return true;
4646
case TPKRangeFilter::EUsageClass::FullUsage:
4747
return false;

‎ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/plain_read_data.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,7 @@ TPlainReadData::TPlainReadData(const std::shared_ptr<TReadContext>& context)
4040
if (GetReadMetadata()->IsWriteConflictable(i.GetInsertWriteId())) {
4141
continue;
4242
}
43-
} else if (GetReadMetadata()->GetPKRangesFilter().IsPortionInPartialUsage(i.GetFirst(), i.GetLast()) ==
44-
TPKRangeFilter::EUsageClass::DontUsage) {
43+
} else if (!GetReadMetadata()->GetPKRangesFilter().IsUsed(i.GetFirst(), i.GetLast())) {
4544
continue;
4645
}
4746
sources.emplace_back(std::make_shared<TCommittedDataSource>(sourceIdx++, i, SpecialReadContext));

‎ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,8 +148,8 @@ class IDataSource: public NCommon::IDataSource {
148148
, Finish(context->GetReadMetadata()->BuildSortedPosition(finish))
149149
, StartReplaceKey(start)
150150
, FinishReplaceKey(finish) {
151-
UsageClass = GetContext()->GetReadMetadata()->GetPKRangesFilter().IsPortionInPartialUsage(GetStartReplaceKey(), GetFinishReplaceKey());
152-
AFL_VERIFY(UsageClass != TPKRangeFilter::EUsageClass::DontUsage);
151+
UsageClass = GetContext()->GetReadMetadata()->GetPKRangesFilter().GetUsageClass(GetStartReplaceKey(), GetFinishReplaceKey());
152+
AFL_VERIFY(UsageClass != TPKRangeFilter::EUsageClass::NoUsage);
153153
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "portions_for_merge")("start", Start.DebugJson())("finish", Finish.DebugJson());
154154
if (Start.IsReverseSort()) {
155155
std::swap(Start, Finish);

‎ydb/core/tx/columnshard/engines/reader/simple_reader/constructor/read_metadata.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
namespace NKikimr::NOlap::NReader::NSimple {
99

1010
std::unique_ptr<TScanIteratorBase> TReadMetadata::StartScan(const std::shared_ptr<TReadContext>& readContext) const {
11-
return std::make_unique<TColumnShardScanIterator>(readContext, readContext->GetReadMetadataPtrVerifiedAs<TReadMetadata>());
11+
return std::make_unique<TColumnShardScanIterator>(readContext);
1212
}
1313

1414
TConclusionStatus TReadMetadata::DoInitCustom(

‎ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::DoGetColumnsFetchingPlan(c
3232
switch (source->GetUsageClass()) {
3333
case TPKRangeFilter::EUsageClass::PartialUsage:
3434
return true;
35-
case TPKRangeFilter::EUsageClass::DontUsage:
35+
case TPKRangeFilter::EUsageClass::NoUsage:
3636
return true;
3737
case TPKRangeFilter::EUsageClass::FullUsage:
3838
return false;

‎ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ using EMemType = NCommon::EMemType;
2121
using TFetchingScriptCursor = NCommon::TFetchingScriptCursor;
2222
using TStepAction = NCommon::TStepAction;
2323

24-
class TSpecialReadContext;
2524

2625
class IFetchingStep: public NCommon::IFetchingStep {
2726
private:

‎ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/plain_read_data.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ TPlainReadData::TPlainReadData(const std::shared_ptr<TReadContext>& context)
1919
insertedPortionsBytes += i->GetTotalBlobBytes();
2020
}
2121

22-
std::make_shared<TPortionDataSource>(sourceIdx++, i, SpecialReadContext);
2322
sources.emplace_back(std::make_shared<TPortionDataSource>(sourceIdx++, i, SpecialReadContext));
2423
}
2524
std::sort(sources.begin(), sources.end(), IDataSource::TCompareStartForScanSequence());

‎ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -255,8 +255,8 @@ class IDataSource: public NCommon::IDataSource {
255255
, Start(context->GetReadMetadata()->IsDescSorted() ? finish : start, context->GetReadMetadata()->IsDescSorted())
256256
, Finish(context->GetReadMetadata()->IsDescSorted() ? start : finish, context->GetReadMetadata()->IsDescSorted()) {
257257
StageData = std::make_unique<TFetchedData>(true);
258-
UsageClass = GetContext()->GetReadMetadata()->GetPKRangesFilter().IsPortionInPartialUsage(start, finish);
259-
AFL_VERIFY(UsageClass != TPKRangeFilter::EUsageClass::DontUsage);
258+
UsageClass = GetContext()->GetReadMetadata()->GetPKRangesFilter().GetUsageClass(start, finish);
259+
AFL_VERIFY(UsageClass != TPKRangeFilter::EUsageClass::NoUsage);
260260
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "portions_for_merge")("start", Start.DebugString())(
261261
"finish", Finish.DebugString());
262262
Y_ABORT_UNLESS(Start.Compare(Finish) != std::partial_ordering::greater);

0 commit comments

Comments
 (0)