Skip to content

Commit b6cb54c

Browse files
kungakunga
committed
Test vector index build serverless consumption (#19656)
1 parent a814333 commit b6cb54c

File tree

8 files changed

+359
-196
lines changed

8 files changed

+359
-196
lines changed

‎ydb/core/tx/datashard/build_index/reshuffle_kmeans.cpp

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -81,9 +81,6 @@ class TReshuffleKMeansScanBase: public TActor<TReshuffleKMeansScanBase>, public
8181

8282
TUploadStatus UploadStatus;
8383

84-
ui64 UploadRows = 0;
85-
ui64 UploadBytes = 0;
86-
8784
TActorId ResponseActorId;
8885
TAutoPtr<TEvDataShard::TEvReshuffleKMeansResponse> Response;
8986

‎ydb/core/tx/schemeshard/schemeshard__serverless_storage_billing.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ struct TSchemeShard::TTxServerlessStorageBilling : public TTransactionBase<TSche
3232
const TSubDomainInfo::TDiskSpaceUsage& spaceUsage = domainDescr->GetDiskSpaceUsage();
3333

3434
if (!Self->IsServerlessDomain(TPath::Init(Self->RootPathId(), Self))) {
35-
LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
35+
LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
3636
"TTxServerlessStorageBilling: unable to make a bill, domain is not a serverless db"
3737
<< ", schemeshardId: " << Self->SelfTabletId()
3838
<< ", domainId: " << Self->ParentDomainId);
@@ -46,7 +46,7 @@ struct TSchemeShard::TTxServerlessStorageBilling : public TTransactionBase<TSche
4646
TimeToNextBill = TimeGrid.GetNext(cur).Start;
4747

4848
if (!Self->AllowServerlessStorageBilling) {
49-
LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
49+
LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
5050
"TTxServerlessStorageBilling: unable to make a bill, AllowServerlessStorageBilling is false"
5151
<< ", schemeshardId: " << Self->SelfTabletId()
5252
<< ", domainId: " << Self->ParentDomainId
@@ -68,7 +68,7 @@ struct TSchemeShard::TTxServerlessStorageBilling : public TTransactionBase<TSche
6868
}
6969

7070
if (!cloud_id || !folder_id || !database_id) {
71-
LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
71+
LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
7272
"TTxServerlessStorageBilling: unable to make a bill, neither cloud_id and nor folder_id nor database_id have found in user attributes at the domain"
7373
<< ", schemeshardId: " << Self->SelfTabletId()
7474
<< ", domainId: " << Self->ParentDomainId

‎ydb/core/tx/schemeshard/schemeshard_billing_helpers.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
namespace NKikimr::NSchemeShard {
88

9-
TBillingStats::TBillingStats(ui64 readRows, ui64 readBytes, ui64 uploadRows, ui64 uploadBytes)
9+
TBillingStats::TBillingStats(ui64 uploadRows, ui64 uploadBytes, ui64 readRows, ui64 readBytes)
1010
: UploadRows{uploadRows}
1111
, UploadBytes{uploadBytes}
1212
, ReadRows{readRows}

‎ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ void TSchemeShard::TIndexBuilder::TTxBase::ApplyBill(NTabletFlatExecutor::TTrans
118118
}
119119

120120
if (!cloud_id || !folder_id || !database_id) {
121-
LOG_N("ApplyBill: unable to make a bill, neither cloud_id and nor folder_id nor database_id have found in user attributes at the domain"
121+
LOG_I("ApplyBill: unable to make a bill, neither cloud_id and nor folder_id nor database_id have found in user attributes at the domain"
122122
<< ", build index operation: " << buildId
123123
<< ", domain: " << domain.PathString()
124124
<< ", domainId: " << buildInfo.DomainPathId
@@ -128,7 +128,7 @@ void TSchemeShard::TIndexBuilder::TTxBase::ApplyBill(NTabletFlatExecutor::TTrans
128128
}
129129

130130
if (!Self->IsServerlessDomain(domain)) {
131-
LOG_N("ApplyBill: unable to make a bill, domain is not a serverless db"
131+
LOG_I("ApplyBill: unable to make a bill, domain is not a serverless db"
132132
<< ", build index operation: " << buildId
133133
<< ", domain: " << domain.PathString()
134134
<< ", domainId: " << buildInfo.DomainPathId

‎ydb/core/tx/schemeshard/ut_helpers/helpers.cpp

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2673,4 +2673,62 @@ namespace NSchemeShardUT_Private {
26732673
columnIds, std::move(matrix), true);
26742674
};
26752675

2676+
void TestCreateServerLessDb(TTestActorRuntime& runtime, TTestEnv& env, ui64& txId, ui64& tenantSchemeShard) {
2677+
TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", "Name: \"ResourceDB\"");
2678+
env.TestWaitNotification(runtime, txId);
2679+
2680+
TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", R"(
2681+
StoragePools {
2682+
Name: "pool-1"
2683+
Kind: "pool-kind-1"
2684+
}
2685+
StoragePools {
2686+
Name: "pool-2"
2687+
Kind: "pool-kind-2"
2688+
}
2689+
PlanResolution: 50
2690+
Coordinators: 1
2691+
Mediators: 1
2692+
TimeCastBucketsPerMediator: 2
2693+
ExternalSchemeShard: true
2694+
Name: "ResourceDB"
2695+
)");
2696+
env.TestWaitNotification(runtime, txId);
2697+
2698+
const auto attrs = AlterUserAttrs({
2699+
{"cloud_id", "CLOUD_ID_VAL"},
2700+
{"folder_id", "FOLDER_ID_VAL"},
2701+
{"database_id", "DATABASE_ID_VAL"},
2702+
});
2703+
TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", Sprintf(R"(
2704+
Name: "ServerLessDB"
2705+
ResourcesDomainKey {
2706+
SchemeShard: %lu
2707+
PathId: 2
2708+
}
2709+
)", TTestTxConfig::SchemeShard), attrs);
2710+
env.TestWaitNotification(runtime, txId);
2711+
2712+
TString alterData = R"(
2713+
PlanResolution: 50
2714+
Coordinators: 1
2715+
Mediators: 1
2716+
TimeCastBucketsPerMediator: 2
2717+
ExternalSchemeShard: true
2718+
ExternalHive: false
2719+
Name: "ServerLessDB"
2720+
StoragePools {
2721+
Name: "pool-1"
2722+
Kind: "pool-kind-1"
2723+
}
2724+
)";
2725+
TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", alterData);
2726+
env.TestWaitNotification(runtime, txId);
2727+
2728+
TestDescribeResult(DescribePath(runtime, "/MyRoot/ServerLessDB"), {
2729+
NLs::PathExist,
2730+
NLs::IsExternalSubDomain("ServerLessDB"),
2731+
NLs::ExtractTenantSchemeshard(&tenantSchemeShard)});
2732+
}
2733+
26762734
}

‎ydb/core/tx/schemeshard/ut_helpers/helpers.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -658,5 +658,7 @@ namespace NSchemeShardUT_Private {
658658

659659
void WriteVectorTableRows(TTestActorRuntime& runtime, ui64 schemeShardId, ui64 txId, const TString & tablePath,
660660
bool withValue, ui32 shard, ui32 min, ui32 max);
661+
662+
void TestCreateServerLessDb(TTestActorRuntime& runtime, TTestEnv& env, ui64& txId, ui64& tenantSchemeShard);
661663

662664
} //NSchemeShardUT_Private

‎ydb/core/tx/schemeshard/ut_index_build/ut_index_build.cpp

Lines changed: 12 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -181,62 +181,11 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) {
181181
TTestEnv env(runtime);
182182
ui64 txId = 100;
183183

184-
TestCreateExtSubDomain(runtime, ++txId, "/MyRoot",
185-
"Name: \"ResourceDB\"");
186-
env.TestWaitNotification(runtime, txId);
187-
188-
TestAlterExtSubDomain(runtime, ++txId, "/MyRoot",
189-
"StoragePools { "
190-
" Name: \"pool-1\" "
191-
" Kind: \"pool-kind-1\" "
192-
"} "
193-
"StoragePools { "
194-
" Name: \"pool-2\" "
195-
" Kind: \"pool-kind-2\" "
196-
"} "
197-
"PlanResolution: 50 "
198-
"Coordinators: 1 "
199-
"Mediators: 1 "
200-
"TimeCastBucketsPerMediator: 2 "
201-
"ExternalSchemeShard: true "
202-
"Name: \"ResourceDB\"");
203-
env.TestWaitNotification(runtime, txId);
204-
205-
const auto attrs = AlterUserAttrs({
206-
{"cloud_id", "CLOUD_ID_VAL"},
207-
{"folder_id", "FOLDER_ID_VAL"},
208-
{"database_id", "DATABASE_ID_VAL"}
209-
});
210-
211-
TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", Sprintf(R"(
212-
Name: "ServerLessDB"
213-
ResourcesDomainKey {
214-
SchemeShard: %lu
215-
PathId: 2
216-
}
217-
)", TTestTxConfig::SchemeShard), attrs);
218-
env.TestWaitNotification(runtime, txId);
219-
220-
TString alterData = TStringBuilder()
221-
<< "PlanResolution: 50 "
222-
<< "Coordinators: 1 "
223-
<< "Mediators: 1 "
224-
<< "TimeCastBucketsPerMediator: 2 "
225-
<< "ExternalSchemeShard: true "
226-
<< "ExternalHive: false "
227-
<< "Name: \"ServerLessDB\" "
228-
<< "StoragePools { "
229-
<< " Name: \"pool-1\" "
230-
<< " Kind: \"pool-kind-1\" "
231-
<< "} ";
232-
TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", alterData);
233-
env.TestWaitNotification(runtime, txId);
184+
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
185+
runtime.SetLogPriority(NKikimrServices::BUILD_INDEX, NLog::PRI_TRACE);
234186

235187
ui64 tenantSchemeShard = 0;
236-
TestDescribeResult(DescribePath(runtime, "/MyRoot/ServerLessDB"),
237-
{NLs::PathExist,
238-
NLs::IsExternalSubDomain("ServerLessDB"),
239-
NLs::ExtractTenantSchemeshard(&tenantSchemeShard)});
188+
TestCreateServerLessDb(runtime, env, txId, tenantSchemeShard);
240189

241190
// Just create main table
242191
TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/ServerLessDB", R"(
@@ -268,9 +217,6 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) {
268217
fnWriteRow(TTestTxConfig::FakeHiveTablets + 6, 1 + delta, 1000 + delta, "aaaa", "Table");
269218
}
270219

271-
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
272-
runtime.SetLogPriority(NKikimrServices::BUILD_INDEX, NLog::PRI_TRACE);
273-
274220
TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/ServerLessDB/Table"),
275221
{NLs::PathExist,
276222
NLs::IndexesCount(0),
@@ -430,6 +376,9 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) {
430376
TTestEnv env(runtime);
431377
ui64 txId = 100;
432378

379+
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
380+
runtime.SetLogPriority(NKikimrServices::BUILD_INDEX, NLog::PRI_TRACE);
381+
433382
SetSplitMergePartCountLimit(&runtime, -1);
434383

435384
// Just create main table
@@ -462,8 +411,6 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) {
462411
fnWriteRow(TTestTxConfig::FakeHiveTablets, 1 + delta, 1000 + delta, longString, "Table");
463412
}
464413

465-
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
466-
467414
TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"),
468415
{NLs::PathExist,
469416
NLs::IndexesCount(0),
@@ -1160,6 +1107,9 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) {
11601107
TTestEnv env(runtime);
11611108
ui64 txId = 100;
11621109

1110+
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
1111+
runtime.SetLogPriority(NKikimrServices::BUILD_INDEX, NLog::PRI_TRACE);
1112+
11631113
// Just create main table
11641114
TestCreateTable(runtime, ++txId, "/MyRoot", R"(
11651115
Name: "Table"
@@ -1189,8 +1139,6 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) {
11891139
fnWriteRow(TTestTxConfig::FakeHiveTablets, 1 + delta, 1000 + delta, "aaaa", "Table");
11901140
}
11911141

1192-
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
1193-
11941142
TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"),
11951143
{NLs::PathExist,
11961144
NLs::IndexesCount(0),
@@ -1225,6 +1173,9 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) {
12251173
TTestEnv env(runtime);
12261174
ui64 txId = 100;
12271175

1176+
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
1177+
runtime.SetLogPriority(NKikimrServices::BUILD_INDEX, NLog::PRI_TRACE);
1178+
12281179
// Just create main table
12291180
TestCreateTable(runtime, ++txId, "/MyRoot", R"(
12301181
Name: "Table"
@@ -1254,8 +1205,6 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) {
12541205
fnWriteRow(TTestTxConfig::FakeHiveTablets, 1 + delta, 1000 + delta, "aaaa", "Table");
12551206
}
12561207

1257-
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
1258-
12591208
TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"),
12601209
{NLs::PathExist,
12611210
NLs::IndexesCount(0),

0 commit comments

Comments
 (0)