Skip to content

Commit 4598038

Browse files
ivanmorozov333ivanmorozov333
authored
add stats test for internal column table from tables store (#6684)
1 parent 34b6297 commit 4598038

File tree

7 files changed

+54
-21
lines changed

7 files changed

+54
-21
lines changed

‎ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,8 @@ TConclusion<std::shared_ptr<arrow::RecordBatch>> ISnapshotSchema::PrepareForModi
7878
for (auto&& i : batch->schema()->fields()) {
7979
AFL_VERIFY(GetIndexInfo().HasColumnName(i->name()));
8080
if (!dstSchema->GetFieldByName(i->name())->Equals(i)) {
81-
return TConclusionStatus::Fail("not equal field types for column '" + i->name() + "'");
81+
return TConclusionStatus::Fail("not equal field types for column '" + i->name() + "': " + i->ToString() + " vs " +
82+
dstSchema->GetFieldByName(i->name())->ToString());
8283
}
8384
if (GetIndexInfo().IsNullableVerified(i->name())) {
8485
continue;

‎ydb/core/tx/columnshard/operations/batch_builder/builder.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
namespace NKikimr::NOlap {
1212

1313
void TBuildBatchesTask::ReplyError(const TString& message) {
14+
AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("problem", "cannot build batch for insert")("reason", message)("data", WriteData.GetWriteMeta().GetLongTxIdOptional());
1415
auto writeDataPtr = std::make_shared<NEvWrite::TWriteData>(std::move(WriteData));
1516
TWritingBuffer buffer(writeDataPtr->GetBlobsAction(), { std::make_shared<TWriteAggregation>(*writeDataPtr) });
1617
auto result = NColumnShard::TEvPrivate::TEvWriteBlobsResult::Error(NKikimrProto::EReplyStatus::CORRUPTED, std::move(buffer), message);

‎ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -125,22 +125,26 @@ bool WriteDataImpl(TTestBasicRuntime& runtime, TActorId& sender, const ui64 shar
125125
}
126126

127127
bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 shardId, const ui64 writeId, const ui64 tableId, const TString& data,
128-
const std::vector<NArrow::NTest::TTestColumn>& ydbSchema, std::vector<ui64>* writeIds, const NEvWrite::EModificationType mType) {
128+
const std::vector<NArrow::NTest::TTestColumn>& ydbSchema, std::vector<ui64>* writeIds, const NEvWrite::EModificationType mType,
129+
const std::set<std::string>& notNullColumns) {
129130
NLongTxService::TLongTxId longTxId;
130131
UNIT_ASSERT(longTxId.ParseString("ydb://long-tx/01ezvvxjdk2hd4vdgjs68knvp8?node_id=1"));
131-
return WriteDataImpl(runtime, sender, shardId, tableId, longTxId, writeId, data, NArrow::MakeArrowSchema(ydbSchema), writeIds, mType);
132-
132+
return WriteDataImpl(
133+
runtime, sender, shardId, tableId, longTxId, writeId, data, NArrow::MakeArrowSchema(ydbSchema, notNullColumns), writeIds, mType);
133134
}
134135

135136
bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 writeId, const ui64 tableId, const TString& data,
136-
const std::vector<NArrow::NTest::TTestColumn>& ydbSchema, bool waitResult, std::vector<ui64>* writeIds, const NEvWrite::EModificationType mType) {
137+
const std::vector<NArrow::NTest::TTestColumn>& ydbSchema, bool waitResult, std::vector<ui64>* writeIds,
138+
const NEvWrite::EModificationType mType, const std::set<std::string>& notNullColumns) {
137139
NLongTxService::TLongTxId longTxId;
138140
UNIT_ASSERT(longTxId.ParseString("ydb://long-tx/01ezvvxjdk2hd4vdgjs68knvp8?node_id=1"));
139141
if (writeIds) {
140-
return WriteDataImpl(runtime, sender, TTestTxConfig::TxTablet0, tableId, longTxId, writeId, data, NArrow::MakeArrowSchema(ydbSchema), writeIds, mType);
142+
return WriteDataImpl(runtime, sender, TTestTxConfig::TxTablet0, tableId, longTxId, writeId, data,
143+
NArrow::MakeArrowSchema(ydbSchema, notNullColumns), writeIds, mType);
141144
}
142145
std::vector<ui64> ids;
143-
return WriteDataImpl(runtime, sender, TTestTxConfig::TxTablet0, tableId, longTxId, writeId, data, NArrow::MakeArrowSchema(ydbSchema), waitResult ? &ids : nullptr, mType);
146+
return WriteDataImpl(runtime, sender, TTestTxConfig::TxTablet0, tableId, longTxId, writeId, data,
147+
NArrow::MakeArrowSchema(ydbSchema, notNullColumns), waitResult ? &ids : nullptr, mType);
144148
}
145149

146150
std::optional<ui64> WriteData(TTestBasicRuntime& runtime, TActorId& sender, const NLongTxService::TLongTxId& longTxId,

‎ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -406,10 +406,12 @@ void PlanSchemaTx(TTestBasicRuntime& runtime, TActorId& sender, NOlap::TSnapshot
406406
void PlanWriteTx(TTestBasicRuntime& runtime, TActorId& sender, NOlap::TSnapshot snap, bool waitResult = true);
407407

408408
bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 shardId, const ui64 writeId, const ui64 tableId, const TString& data,
409-
const std::vector<NArrow::NTest::TTestColumn>& ydbSchema, std::vector<ui64>* writeIds, const NEvWrite::EModificationType mType = NEvWrite::EModificationType::Upsert);
409+
const std::vector<NArrow::NTest::TTestColumn>& ydbSchema, std::vector<ui64>* writeIds,
410+
const NEvWrite::EModificationType mType = NEvWrite::EModificationType::Upsert, const std::set<std::string>& notNullColumns = {});
410411

411412
bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 writeId, const ui64 tableId, const TString& data,
412-
const std::vector<NArrow::NTest::TTestColumn>& ydbSchema, bool waitResult = true, std::vector<ui64>* writeIds = nullptr, const NEvWrite::EModificationType mType = NEvWrite::EModificationType::Upsert);
413+
const std::vector<NArrow::NTest::TTestColumn>& ydbSchema, bool waitResult = true, std::vector<ui64>* writeIds = nullptr,
414+
const NEvWrite::EModificationType mType = NEvWrite::EModificationType::Upsert, const std::set<std::string>& notNullColumns = {});
413415

414416
std::optional<ui64> WriteData(TTestBasicRuntime& runtime, TActorId& sender, const NLongTxService::TLongTxId& longTxId,
415417
ui64 tableId, const ui64 writePartId, const TString& data,

‎ydb/core/tx/columnshard/test_helper/helper.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ class TTestColumn {
5151
YDB_ACCESSOR_DEF(TString, Name);
5252
YDB_ACCESSOR_DEF(NScheme::TTypeInfo, Type);
5353
YDB_ACCESSOR_DEF(TString, StorageId);
54+
5455
public:
5556
explicit TTestColumn(const TString& name, const NScheme::TTypeInfo& type)
5657
: Name(name)

‎ydb/core/tx/schemeshard/schemeshard_path_describer.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -486,6 +486,8 @@ void TPathDescriber::DescribeColumnTable(TPathId pathId, TPathElement::TPtr path
486486
}
487487
if (tableInfo->GetStats().TableStats.contains(pathId)) {
488488
FillTableStats(*pathDescription, tableInfo->GetStats().TableStats.at(pathId));
489+
} else {
490+
FillTableStats(*pathDescription, TPartitionStats());
489491
}
490492
}
491493
}

‎ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp

Lines changed: 34 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h>
22
#include <ydb/core/tx/columnshard/columnshard.h>
33
#include <ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h>
4+
#include <ydb/core/tx/columnshard/hooks/testing/controller.h>
45
#include <ydb/core/formats/arrow/arrow_helpers.h>
56
#include <ydb/core/formats/arrow/arrow_batch_builder.h>
67

@@ -634,17 +635,17 @@ Y_UNIT_TEST_SUITE(TOlap) {
634635
env.TestWaitNotification(runtime, txId);
635636
}
636637

637-
// TODO: AlterTiers
638-
// negatives for store: disallow alters
639-
// negatives for table: wrong tiers count, wrong tiers, wrong eviction column, wrong eviction values,
640-
// different TTL columns in tiers
641-
#if 0
642638
Y_UNIT_TEST(StoreStats) {
643639
TTestBasicRuntime runtime;
644640
TTestEnv env(runtime);
645641
runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_DEBUG);
646642
runtime.UpdateCurrentTime(TInstant::Now() - TDuration::Seconds(600));
647643

644+
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
645+
csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1));
646+
csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1));
647+
csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30);
648+
648649
// disable stats batching
649650
auto& appData = runtime.GetAppData();
650651
appData.SchemeShardConfig.SetStatsBatchTimeoutMs(0);
@@ -690,6 +691,16 @@ Y_UNIT_TEST_SUITE(TOlap) {
690691
UNIT_ASSERT(shardId);
691692
UNIT_ASSERT(pathId);
692693
UNIT_ASSERT(planStep);
694+
{
695+
auto description = DescribePrivatePath(runtime, TTestTxConfig::SchemeShard, "/MyRoot/OlapStore/ColumnTable", true, true);
696+
Cerr << description.DebugString() << Endl;
697+
auto& tabletStats = description.GetPathDescription().GetTableStats();
698+
699+
UNIT_ASSERT(description.GetPathDescription().HasTableStats());
700+
UNIT_ASSERT_EQUAL(tabletStats.GetRowCount(), 0);
701+
UNIT_ASSERT_EQUAL(tabletStats.GetDataSize(), 0);
702+
}
703+
693704

694705
ui32 rowsInBatch = 100000;
695706

@@ -702,7 +713,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
702713
TSet<ui64> txIds;
703714
for (ui32 i = 0; i < 10; ++i) {
704715
std::vector<ui64> writeIds;
705-
NTxUT::WriteData(runtime, sender, shardId, ++writeId, pathId, data, defaultYdbSchema, &writeIds);
716+
NTxUT::WriteData(runtime, sender, shardId, ++writeId, pathId, data, defaultYdbSchema, &writeIds, NEvWrite::EModificationType::Upsert, { "timestamp" });
706717
NTxUT::ProposeCommit(runtime, sender, shardId, ++txId, writeIds);
707718
txIds.insert(txId);
708719
}
@@ -714,16 +725,28 @@ Y_UNIT_TEST_SUITE(TOlap) {
714725

715726
// trigger periodic stats at shard (after timeout)
716727
std::vector<ui64> writeIds;
717-
NTxUT::WriteData(runtime, sender, shardId, ++writeId, pathId, data, defaultYdbSchema, &writeIds);
728+
NTxUT::WriteData(runtime, sender, shardId, ++writeId, pathId, data, defaultYdbSchema, &writeIds, NEvWrite::EModificationType::Upsert, { "timestamp" });
718729
NTxUT::ProposeCommit(runtime, sender, shardId, ++txId, writeIds);
719730
NTxUT::PlanCommit(runtime, sender, shardId, ++planStep, {txId});
720731
}
732+
csController->WaitIndexation(TDuration::Seconds(5));
733+
{
734+
auto description = DescribePrivatePath(runtime, TTestTxConfig::SchemeShard, "/MyRoot/OlapStore", true, true);
735+
auto& tabletStats = description.GetPathDescription().GetTableStats();
721736

722-
auto description = DescribePrivatePath(runtime, TTestTxConfig::SchemeShard, "/MyRoot/OlapStore", true, true);
723-
auto& tabletStats = description.GetPathDescription().GetTableStats();
737+
UNIT_ASSERT_GT(tabletStats.GetRowCount(), 0);
738+
UNIT_ASSERT_GT(tabletStats.GetDataSize(), 0);
739+
}
740+
741+
{
742+
auto description = DescribePrivatePath(runtime, TTestTxConfig::SchemeShard, "/MyRoot/OlapStore/ColumnTable", true, true);
743+
Cerr << description.DebugString() << Endl;
744+
auto& tabletStats = description.GetPathDescription().GetTableStats();
745+
746+
UNIT_ASSERT_GT(tabletStats.GetRowCount(), 0);
747+
UNIT_ASSERT_GT(tabletStats.GetDataSize(), 0);
748+
}
724749

725-
UNIT_ASSERT_GT(tabletStats.GetRowCount(), 0);
726-
UNIT_ASSERT_GT(tabletStats.GetDataSize(), 0);
727750
#if 0
728751
TestDropColumnTable(runtime, ++txId, "/MyRoot/OlapStore", "ColumnTable");
729752
env.TestWaitNotification(runtime, txId);
@@ -738,5 +761,4 @@ Y_UNIT_TEST_SUITE(TOlap) {
738761
TestLsPathId(runtime, 2, NLs::PathStringEqual(""));
739762
#endif
740763
}
741-
#endif
742764
}

0 commit comments

Comments
 (0)