24-3 Report index sizes by type (#8325)

Author: kunga

ydb/core/tablet_flat/flat_database.cpp

@@ -494,7 +494,8 @@ ui64 TDatabase::GetTableMemOpsCount(ui32 tableId) const {
 }
 
 ui64 TDatabase::GetTableIndexSize(ui32 tableId) const {
-    return Require(tableId)->Stat().Parts.IndexBytes;
+    const auto& partStats = Require(tableId)->Stat().Parts;
+    return partStats.FlatIndexBytes + partStats.BTreeIndexBytes;
 }
 
 ui64 TDatabase::GetTableSearchHeight(ui32 tableId) const {

ydb/core/tablet_flat/flat_dbase_misc.h

@@ -14,7 +14,7 @@ namespace NTable {
 
         void Describe(IOutputStream &out) const noexcept
         {
-            const ui64 sys = Parts.IndexBytes + Parts.ByKeyBytes + Parts.OtherBytes;
+            const ui64 sys = Parts.FlatIndexBytes + Parts.BTreeIndexBytes + Parts.ByKeyBytes + Parts.OtherBytes;
 
             out
                 << "DBase{" << Tables << "t " << Parts.PartsCount << "p"

ydb/core/tablet_flat/flat_executor.cpp

@@ -3590,7 +3590,9 @@ void TExecutor::UpdateCounters(const TActorContext &ctx) {
             { /* Memory consumption of common for leader and follower components */
                 Counters->Simple()[TExecutorCounters::DB_WARM_BYTES].Set(dbCounters.MemTableBytes);
                 Counters->Simple()[TExecutorCounters::DB_META_BYTES].Set(Stats->PacksMetaBytes);
-                Counters->Simple()[TExecutorCounters::DB_INDEX_BYTES].Set(dbCounters.Parts.IndexBytes);
+                Counters->Simple()[TExecutorCounters::DB_FLAT_INDEX_BYTES].Set(dbCounters.Parts.FlatIndexBytes);
+                Counters->Simple()[TExecutorCounters::DB_B_TREE_INDEX_BYTES].Set(dbCounters.Parts.BTreeIndexBytes);
+                Counters->Simple()[TExecutorCounters::DB_INDEX_BYTES].Set(dbCounters.Parts.FlatIndexBytes + dbCounters.Parts.BTreeIndexBytes);
                 Counters->Simple()[TExecutorCounters::DB_OTHER_BYTES].Set(dbCounters.Parts.OtherBytes);
                 Counters->Simple()[TExecutorCounters::DB_BYKEY_BYTES].Set(dbCounters.Parts.ByKeyBytes);
                 Counters->Simple()[TExecutorCounters::USED_TABLET_MEMORY].Set(UsedTabletMemory);

ydb/core/tablet_flat/flat_executor_counters.h

@@ -62,6 +62,8 @@ namespace NTabletFlatExecutor {
     XX(CONSUMED_STORAGE, "ConsumedStorage") \
     XX(CONSUMED_MEMORY, "ConsumedMemory") \
     XX(COMPACTION_READ_IN_FLY, "CompactionReadInFly") \
+    XX(DB_FLAT_INDEX_BYTES, "DbFlatIndexBytes") \
+    XX(DB_B_TREE_INDEX_BYTES, "DbBTreeIndexBytes") \
 
 // don't change order!
 #define FLAT_EXECUTOR_CUMULATIVE_COUNTERS_MAP(XX) \

ydb/core/tablet_flat/flat_table.cpp

@@ -1415,7 +1415,11 @@ void TTable::SetTableObserver(TIntrusivePtr<ITableObserver> ptr) noexcept
 void TPartStats::Add(const TPartView& partView)
 {
     PartsCount += 1;
-    IndexBytes += partView->IndexesRawSize;
+    if (partView->IndexPages.HasBTree()) {
+        BTreeIndexBytes += partView->IndexesRawSize;
+    } else {
+        FlatIndexBytes += partView->IndexesRawSize;
+    }
     ByKeyBytes += partView->ByKey ? partView->ByKey->Raw.size() : 0;
     PlainBytes += partView->Stat.Bytes;
     CodedBytes += partView->Stat.Coded;
@@ -1434,7 +1438,11 @@ void TPartStats::Add(const TPartView& partView)
 bool TPartStats::Remove(const TPartView& partView)
 {
     NUtil::SubSafe(PartsCount, ui64(1));
-    NUtil::SubSafe(IndexBytes, partView->IndexesRawSize);
+    if (partView->IndexPages.HasBTree()) {
+        NUtil::SubSafe(BTreeIndexBytes, partView->IndexesRawSize);
+    } else {
+        NUtil::SubSafe(FlatIndexBytes, partView->IndexesRawSize);
+    }
     NUtil::SubSafe(ByKeyBytes, partView->ByKey ? partView->ByKey->Raw.size() : 0);
     NUtil::SubSafe(PlainBytes, partView->Stat.Bytes);
     NUtil::SubSafe(CodedBytes, partView->Stat.Coded);
@@ -1463,7 +1471,8 @@ bool TPartStats::Remove(const TPartView& partView)
 TPartStats& TPartStats::operator+=(const TPartStats& rhs)
 {
     PartsCount += rhs.PartsCount;
-    IndexBytes += rhs.IndexBytes;
+    FlatIndexBytes += rhs.FlatIndexBytes;
+    BTreeIndexBytes += rhs.BTreeIndexBytes;
     OtherBytes += rhs.OtherBytes;
     ByKeyBytes += rhs.ByKeyBytes;
     PlainBytes += rhs.PlainBytes;
@@ -1480,7 +1489,8 @@ TPartStats& TPartStats::operator+=(const TPartStats& rhs)
 TPartStats& TPartStats::operator-=(const TPartStats& rhs)
 {
     NUtil::SubSafe(PartsCount, rhs.PartsCount);
-    NUtil::SubSafe(IndexBytes, rhs.IndexBytes);
+    NUtil::SubSafe(FlatIndexBytes, rhs.FlatIndexBytes);
+    NUtil::SubSafe(BTreeIndexBytes, rhs.BTreeIndexBytes);
     NUtil::SubSafe(OtherBytes, rhs.OtherBytes);
     NUtil::SubSafe(ByKeyBytes, rhs.ByKeyBytes);
     NUtil::SubSafe(PlainBytes, rhs.PlainBytes);

ydb/core/tablet_flat/flat_table_stats.h

@@ -8,7 +8,8 @@ namespace NTable {
 
     struct TPartStats {
         ui64 PartsCount = 0;    /* Total used TPart units in db    */
-        ui64 IndexBytes = 0;
+        ui64 FlatIndexBytes = 0;
+        ui64 BTreeIndexBytes = 0;
         ui64 OtherBytes = 0;    /* Other metadata and sys. indexes */
         ui64 ByKeyBytes = 0;
         ui64 PlainBytes = 0;    /* Plain data pages size           */

ydb/core/tablet_flat/ut/ut_db_iface.cpp

@@ -276,7 +276,8 @@ Y_UNIT_TEST_SUITE(DBase) {
         UNIT_ASSERT(me->Counters().Parts.RowsErase == 0);
         UNIT_ASSERT(me->Counters().Parts.PartsCount == 0);
         UNIT_ASSERT(me->Counters().Parts.PlainBytes == 0);
-        UNIT_ASSERT(me->Counters().Parts.IndexBytes == 0);
+        UNIT_ASSERT(me->Counters().Parts.FlatIndexBytes == 0);
+        UNIT_ASSERT(me->Counters().Parts.BTreeIndexBytes == 0);
         UNIT_ASSERT(me->Counters().Parts.OtherBytes == 0);
     }