Skip to content

Commit 75ab2b2

Browse files
ivanmorozov333ivanmorozov333
authored
improve performace for stat requests (sysview) (#7211)
1 parent 0bc0896 commit 75ab2b2

File tree

2 files changed

+76
-11
lines changed

2 files changed

+76
-11
lines changed

‎ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp

Lines changed: 50 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,19 @@ namespace NKikimr::NOlap::NReader::NSysView::NChunks {
66

77
void TStatsIterator::AppendStats(const std::vector<std::unique_ptr<arrow::ArrayBuilder>>& builders, const TPortionInfo& portion) const {
88
auto portionSchema = ReadMetadata->GetLoadSchemaVerified(portion);
9-
const std::string prod = ::ToString(portion.GetMeta().Produced);
9+
auto it = PortionType.find(portion.GetMeta().Produced);
10+
if (it == PortionType.end()) {
11+
it = PortionType.emplace(portion.GetMeta().Produced, ::ToString(portion.GetMeta().Produced)).first;
12+
}
13+
const arrow::util::string_view prodView = it->second.GetView();
1014
const bool activity = !portion.IsRemovedFor(ReadMetadata->GetRequestSnapshot());
15+
static const TString ConstantEntityIsColumn = "COL";
16+
static const arrow::util::string_view ConstantEntityIsColumnView =
17+
arrow::util::string_view(ConstantEntityIsColumn.data(), ConstantEntityIsColumn.size());
18+
static const TString ConstantEntityIsIndex = "IDX";
19+
static const arrow::util::string_view ConstantEntityIsIndexView =
20+
arrow::util::string_view(ConstantEntityIsIndex.data(), ConstantEntityIsIndex.size());
21+
auto& entityStorages = EntityStorageNames[portion.GetMeta().GetTierName()];
1122
{
1223
std::vector<const TColumnRecord*> records;
1324
for (auto&& r : portion.Records) {
@@ -16,26 +27,54 @@ void TStatsIterator::AppendStats(const std::vector<std::unique_ptr<arrow::ArrayB
1627
if (Reverse) {
1728
std::reverse(records.begin(), records.end());
1829
}
30+
THashMap<ui32, TString> blobsIds;
31+
std::optional<ui32> lastColumnId;
32+
arrow::util::string_view lastColumnName;
33+
arrow::util::string_view lastTierName;
1934
for (auto&& r : records) {
2035
NArrow::Append<arrow::UInt64Type>(*builders[0], portion.GetPathId());
21-
NArrow::Append<arrow::StringType>(*builders[1], prod);
36+
NArrow::Append<arrow::StringType>(*builders[1], prodView);
2237
NArrow::Append<arrow::UInt64Type>(*builders[2], ReadMetadata->TabletId);
2338
NArrow::Append<arrow::UInt64Type>(*builders[3], r->GetMeta().GetNumRows());
2439
NArrow::Append<arrow::UInt64Type>(*builders[4], r->GetMeta().GetRawBytes());
2540
NArrow::Append<arrow::UInt64Type>(*builders[5], portion.GetPortionId());
2641
NArrow::Append<arrow::UInt64Type>(*builders[6], r->GetChunkIdx());
27-
NArrow::Append<arrow::StringType>(*builders[7], ReadMetadata->GetColumnNameDef(r->GetColumnId()).value_or("undefined"));
42+
if (!lastColumnId || *lastColumnId != r->GetColumnId()) {
43+
{
44+
auto it = ColumnNamesById.find(r->GetColumnId());
45+
if (it == ColumnNamesById.end()) {
46+
it =
47+
ColumnNamesById.emplace(r->GetColumnId(), portionSchema->GetFieldByColumnIdVerified(r->GetColumnId())->name()).first;
48+
}
49+
lastColumnName = it->second.GetView();
50+
}
51+
{
52+
auto it = entityStorages.find(r->GetColumnId());
53+
if (it == entityStorages.end()) {
54+
it = entityStorages.emplace(r->GetColumnId(),
55+
portionSchema->GetIndexInfo().GetEntityStorageId(r->GetColumnId(), portion.GetMeta().GetTierName())).first;
56+
}
57+
lastTierName = it->second.GetView();
58+
}
59+
lastColumnId = r->GetColumnId();
60+
}
61+
NArrow::Append<arrow::StringType>(*builders[7], lastColumnName);
2862
NArrow::Append<arrow::UInt32Type>(*builders[8], r->GetColumnId());
29-
std::string blobIdString = portion.GetBlobId(r->GetBlobRange().GetBlobIdxVerified()).ToStringLegacy();
30-
NArrow::Append<arrow::StringType>(*builders[9], blobIdString);
63+
{
64+
auto itBlobIdString = blobsIds.find(r->GetBlobRange().GetBlobIdxVerified());
65+
if (itBlobIdString == blobsIds.end()) {
66+
itBlobIdString = blobsIds.emplace(
67+
r->GetBlobRange().GetBlobIdxVerified(), portion.GetBlobId(r->GetBlobRange().GetBlobIdxVerified()).ToStringLegacy()).first;
68+
}
69+
NArrow::Append<arrow::StringType>(
70+
*builders[9], arrow::util::string_view(itBlobIdString->second.data(), itBlobIdString->second.size()));
71+
}
3172
NArrow::Append<arrow::UInt64Type>(*builders[10], r->BlobRange.Offset);
3273
NArrow::Append<arrow::UInt64Type>(*builders[11], r->BlobRange.Size);
3374
NArrow::Append<arrow::BooleanType>(*builders[12], activity);
3475

35-
const auto tierName = portionSchema->GetIndexInfo().GetEntityStorageId(r->GetColumnId(), portion.GetMeta().GetTierName());
36-
std::string strTierName(tierName.data(), tierName.size());
37-
NArrow::Append<arrow::StringType>(*builders[13], strTierName);
38-
NArrow::Append<arrow::StringType>(*builders[14], "COL");
76+
NArrow::Append<arrow::StringType>(*builders[13], arrow::util::string_view(lastTierName.data(), lastTierName.size()));
77+
NArrow::Append<arrow::StringType>(*builders[14], ConstantEntityIsColumnView);
3978
}
4079
}
4180
{
@@ -48,7 +87,7 @@ void TStatsIterator::AppendStats(const std::vector<std::unique_ptr<arrow::ArrayB
4887
}
4988
for (auto&& r : indexes) {
5089
NArrow::Append<arrow::UInt64Type>(*builders[0], portion.GetPathId());
51-
NArrow::Append<arrow::StringType>(*builders[1], prod);
90+
NArrow::Append<arrow::StringType>(*builders[1], prodView);
5291
NArrow::Append<arrow::UInt64Type>(*builders[2], ReadMetadata->TabletId);
5392
NArrow::Append<arrow::UInt64Type>(*builders[3], r->GetRecordsCount());
5493
NArrow::Append<arrow::UInt64Type>(*builders[4], r->GetRawBytes());
@@ -70,7 +109,7 @@ void TStatsIterator::AppendStats(const std::vector<std::unique_ptr<arrow::ArrayB
70109
const auto tierName = portionSchema->GetIndexInfo().GetEntityStorageId(r->GetIndexId(), portion.GetMeta().GetTierName());
71110
std::string strTierName(tierName.data(), tierName.size());
72111
NArrow::Append<arrow::StringType>(*builders[13], strTierName);
73-
NArrow::Append<arrow::StringType>(*builders[14], "IDX");
112+
NArrow::Append<arrow::StringType>(*builders[14], ConstantEntityIsIndexView);
74113
}
75114
}
76115
}

‎ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,32 @@ class TReadStatsMetadata: public NAbstract::TReadStatsMetadata, std::enable_shar
2828

2929
class TStatsIterator: public NAbstract::TStatsIterator<NKikimr::NSysView::Schema::PrimaryIndexStats> {
3030
private:
31+
class TViewContainer {
32+
private:
33+
TString Data;
34+
std::string STLData;
35+
arrow::util::string_view View;
36+
37+
public:
38+
const arrow::util::string_view& GetView() const {
39+
return View;
40+
}
41+
42+
TViewContainer(const TString& data)
43+
: Data(data)
44+
, View(arrow::util::string_view(Data.data(), Data.size())) {
45+
}
46+
47+
TViewContainer(const std::string& data)
48+
: STLData(data)
49+
, View(arrow::util::string_view(STLData.data(), STLData.size())) {
50+
}
51+
};
52+
53+
mutable THashMap<ui32, TViewContainer> ColumnNamesById;
54+
mutable THashMap<NPortion::EProduced, TViewContainer> PortionType;
55+
mutable THashMap<TString, THashMap<ui32, TViewContainer>> EntityStorageNames;
56+
3157
using TBase = NAbstract::TStatsIterator<NKikimr::NSysView::Schema::PrimaryIndexStats>;
3258
virtual bool AppendStats(const std::vector<std::unique_ptr<arrow::ArrayBuilder>>& builders, NAbstract::TGranuleMetaView& granule) const override;
3359
virtual ui32 PredictRecordsCount(const NAbstract::TGranuleMetaView& granule) const override;

0 commit comments

Comments
 (0)