Skip to content

Commit 6cd8814

Browse files
MBkktMBkkt
authored
Refactor build index (#7658)
1 parent 5d2be00 commit 6cd8814

31 files changed

+992
-1166
lines changed

‎ydb/core/protos/flat_scheme_op.proto

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1679,7 +1679,7 @@ message TIndexBuildConfig {
16791679
message TIndexBuildControl {
16801680
optional string TablePath = 1;
16811681
optional string IndexName = 2;
1682-
optional uint64 SnaphotTxId = 3;
1682+
optional uint64 SnapshotTxId = 3;
16831683
optional uint64 BuildIndexId = 4;
16841684
}
16851685

‎ydb/core/protos/index_builder.proto

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,3 +106,14 @@ message TEvListResponse {
106106
repeated TIndexBuild Entries = 3;
107107
optional string NextPageToken = 4;
108108
}
109+
110+
enum EBuildStatus {
111+
INVALID = 0;
112+
ACCEPTED = 1;
113+
IN_PROGRESS = 2;
114+
DONE = 3;
115+
116+
ABORTED = 4;
117+
BUILD_ERROR = 5;
118+
BAD_REQUEST = 6;
119+
}

‎ydb/core/protos/tx_datashard.proto

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1429,23 +1429,12 @@ message TEvBuildIndexProgressResponse {
14291429
optional uint64 OwnerId = 3;
14301430
optional uint64 PathId = 4;
14311431

1432-
enum EStatus {
1433-
INVALID = 0;
1434-
ACCEPTED = 1;
1435-
INPROGRESS = 2;
1436-
DONE = 3;
1437-
1438-
ABORTED = 4;
1439-
BUILD_ERROR = 5;
1440-
BAD_REQUEST = 6;
1441-
}
1442-
1443-
optional EStatus Status = 5;
1432+
optional NKikimrIndexBuilder.EBuildStatus Status = 5;
14441433

14451434
optional Ydb.StatusIds.StatusCode UploadStatus = 6;
14461435
repeated Ydb.Issue.IssueMessage Issues = 7;
14471436

1448-
optional NKikimrTx.TKeyRange RequestedKeyRange = 8;
1437+
reserved 8;
14491438
optional bytes LastKeyAck = 9;
14501439

14511440
optional uint64 RowsDelta = 10;
@@ -1480,15 +1469,7 @@ message TEvSampleKResponse {
14801469
optional uint64 TabletId = 2;
14811470
optional NKikimrProto.TPathID PathId = 3;
14821471

1483-
enum EStatus {
1484-
INVALID = 0;
1485-
DONE = 1;
1486-
1487-
ABORTED = 2;
1488-
BAD_REQUEST = 3;
1489-
}
1490-
1491-
optional EStatus Status = 4;
1472+
optional NKikimrIndexBuilder.EBuildStatus Status = 4;
14921473
repeated Ydb.Issue.IssueMessage Issues = 5;
14931474

14941475
optional NKikimrTx.TKeyRange RequestedKeyRange = 6;

‎ydb/core/tx/datashard/build_index.cpp

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -373,14 +373,14 @@ class TBuildScanUpload: public TActor<TBuildScanUpload<Activity>>, public NTable
373373
progress->Record.SetRequestSeqNoRound(SeqNo.Round);
374374

375375
if (abort != EAbort::None) {
376-
progress->Record.SetStatus(NKikimrTxDataShard::TEvBuildIndexProgressResponse::ABORTED);
376+
progress->Record.SetStatus(NKikimrIndexBuilder::EBuildStatus::ABORTED);
377377
UploadStatus.Issues.AddIssue(NYql::TIssue("Aborted by scan host env"));
378378

379379
LOG_W(Debug());
380380
} else if (!UploadStatus.IsSuccess()) {
381-
progress->Record.SetStatus(NKikimrTxDataShard::TEvBuildIndexProgressResponse::BUILD_ERROR);
381+
progress->Record.SetStatus(NKikimrIndexBuilder::EBuildStatus::BUILD_ERROR);
382382
} else {
383-
progress->Record.SetStatus(NKikimrTxDataShard::TEvBuildIndexProgressResponse::DONE);
383+
progress->Record.SetStatus(NKikimrIndexBuilder::EBuildStatus::DONE);
384384
}
385385

386386
UploadStatusToMessage(progress->Record);
@@ -478,12 +478,13 @@ class TBuildScanUpload: public TActor<TBuildScanUpload<Activity>>, public NTable
478478
progress->Record.SetRequestSeqNoGeneration(SeqNo.Generation);
479479
progress->Record.SetRequestSeqNoRound(SeqNo.Round);
480480

481-
progress->Record.SetLastKeyAck(TSerializedCellVec::Serialize(LastUploadedKey.GetCells()));
481+
// TODO(mbkkt) ReleaseBuffer isn't possible, we use LastUploadedKey for logging
482+
progress->Record.SetLastKeyAck(LastUploadedKey.GetBuffer());
482483
progress->Record.SetRowsDelta(WriteBuf.GetRows());
483484
progress->Record.SetBytesDelta(WriteBuf.GetBytes());
484485
WriteBuf.Clear();
485486

486-
progress->Record.SetStatus(NKikimrTxDataShard::TEvBuildIndexProgressResponse::INPROGRESS);
487+
progress->Record.SetStatus(NKikimrIndexBuilder::EBuildStatus::IN_PROGRESS);
487488
UploadStatusToMessage(progress->Record);
488489

489490
ctx.Send(ProgressActorId, progress.Release());
@@ -666,14 +667,13 @@ void TDataShard::HandleSafe(TEvDataShard::TEvBuildIndexCreateRequest::TPtr& ev,
666667
auto response = MakeHolder<TEvDataShard::TEvBuildIndexProgressResponse>();
667668
response->Record.SetBuildIndexId(record.GetBuildIndexId());
668669
response->Record.SetTabletId(TabletID());
669-
response->Record.SetStatus(NKikimrTxDataShard::TEvBuildIndexProgressResponse::ACCEPTED);
670+
response->Record.SetStatus(NKikimrIndexBuilder::EBuildStatus::ACCEPTED);
670671

671672
TScanRecord::TSeqNo seqNo = {record.GetSeqNoGeneration(), record.GetSeqNoRound()};
672673
response->Record.SetRequestSeqNoGeneration(seqNo.Generation);
673674
response->Record.SetRequestSeqNoRound(seqNo.Round);
674-
675675
auto badRequest = [&](const TString& error) {
676-
response->Record.SetStatus(NKikimrTxDataShard::TEvBuildIndexProgressResponse::BAD_REQUEST);
676+
response->Record.SetStatus(NKikimrIndexBuilder::EBuildStatus::BAD_REQUEST);
677677
auto issue = response->Record.AddIssues();
678678
issue->set_severity(NYql::TSeverityIds::S_ERROR);
679679
issue->set_message(error);

‎ydb/core/tx/datashard/check_distributed_erase_tx_unit.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,9 @@ class TCheckDistributedEraseTxUnit : public TExecutionUnit {
8888
}
8989

9090
if (indexCells.GetCells().size() != static_cast<ui32>(eraseTx->GetIndexColumnIds().size())) {
91-
return buildUnsuccessfulResult("Cell count doesn't match row scheme");
91+
return buildUnsuccessfulResult(TStringBuilder() << "Cell count doesn't match row scheme"
92+
<< ": got " << indexCells.GetCells().size()
93+
<< ", expected " << eraseTx->GetIndexColumnIds().size());
9294
}
9395
}
9496

‎ydb/core/tx/datashard/datashard_common_upload.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,9 @@ bool TCommonUploadOps<TEvRequest, TEvResponse>::Execute(TDataShard* self, TTrans
120120
if (keyCells.GetCells().size() != tableInfo.KeyColumnTypes.size() ||
121121
valueCells.GetCells().size() != valueCols.size())
122122
{
123-
SetError(NKikimrTxDataShard::TError::SCHEME_ERROR, "Cell count doesn't match row scheme");
123+
SetError(NKikimrTxDataShard::TError::SCHEME_ERROR, TStringBuilder() << "Cell count doesn't match row scheme"
124+
<< ": got keys " << keyCells.GetCells().size() << ", values " << valueCells.GetCells().size()
125+
<< "; expected keys " << tableInfo.KeyColumnTypes.size() << ", values " << valueCols.size());
124126
return true;
125127
}
126128

‎ydb/core/tx/datashard/datashard_direct_erase.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,9 @@ TDirectTxErase::EStatus TDirectTxErase::CheckedExecute(
9494

9595
if (keyCells.GetCells().size() != tableInfo.KeyColumnTypes.size()) {
9696
status = NKikimrTxDataShard::TEvEraseRowsResponse::SCHEME_ERROR;
97-
error = "Cell count doesn't match row scheme";
97+
error = TStringBuilder() << "Cell count doesn't match row scheme"
98+
<< ": got " << keyCells.GetCells().size()
99+
<< ", expected " << tableInfo.KeyColumnTypes.size();
98100
return EStatus::Error;
99101
}
100102

‎ydb/core/tx/datashard/datashard_ut_build_index.cpp

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,15 @@
77
#include <ydb/core/tx/tx_proxy/proxy.h>
88
#include <ydb/core/tx/tx_proxy/upload_rows.h>
99
#include <ydb/core/testlib/actors/block_events.h>
10+
#include <ydb/core/protos/index_builder.pb.h>
1011

1112
#include <ydb/library/yql/public/issue/yql_issue_message.h>
1213

1314
#include <library/cpp/testing/unittest/registar.h>
1415

1516
template <>
16-
inline void Out<NKikimrTxDataShard::TEvBuildIndexProgressResponse::EStatus>
17-
(IOutputStream& o, NKikimrTxDataShard::TEvBuildIndexProgressResponse::EStatus status)
18-
{
19-
o << NKikimrTxDataShard::TEvBuildIndexProgressResponse::EStatus_Name(status);
17+
inline void Out<NKikimrIndexBuilder::EBuildStatus>(IOutputStream& o, NKikimrIndexBuilder::EBuildStatus status) {
18+
o << NKikimrIndexBuilder::EBuildStatus_Name(status);
2019
}
2120

2221
namespace NKikimr {
@@ -31,7 +30,7 @@ Y_UNIT_TEST_SUITE(TTxDataShardBuildIndexScan) {
3130
static void DoBuildIndex(Tests::TServer::TPtr server, TActorId sender,
3231
const TString& tableFrom, const TString& tableTo,
3332
const TRowVersion& snapshot,
34-
const NKikimrTxDataShard::TEvBuildIndexProgressResponse::EStatus& expected) {
33+
const NKikimrIndexBuilder::EBuildStatus& expected) {
3534
auto &runtime = *server->GetRuntime();
3635
TVector<ui64> datashards = GetTableShards(server, sender, tableFrom);
3736
TTableId tableId = ResolveTableId(server, sender, tableFrom);
@@ -58,14 +57,14 @@ Y_UNIT_TEST_SUITE(TTxDataShardBuildIndexScan) {
5857
TAutoPtr<IEventHandle> handle;
5958
auto reply = runtime.GrabEdgeEventRethrow<TEvDataShard::TEvBuildIndexProgressResponse>(handle);
6059

61-
if (expected == NKikimrTxDataShard::TEvBuildIndexProgressResponse::DONE
62-
&& reply->Record.GetStatus() == NKikimrTxDataShard::TEvBuildIndexProgressResponse::ACCEPTED) {
60+
if (expected == NKikimrIndexBuilder::EBuildStatus::DONE
61+
&& reply->Record.GetStatus() == NKikimrIndexBuilder::EBuildStatus::ACCEPTED) {
6362
Cerr << "skip ACCEPTED" << Endl;
6463
continue;
6564
}
6665

67-
if (expected != NKikimrTxDataShard::TEvBuildIndexProgressResponse::INPROGRESS
68-
&& reply->Record.GetStatus() == NKikimrTxDataShard::TEvBuildIndexProgressResponse::INPROGRESS) {
66+
if (expected != NKikimrIndexBuilder::EBuildStatus::IN_PROGRESS
67+
&& reply->Record.GetStatus() == NKikimrIndexBuilder::EBuildStatus::IN_PROGRESS) {
6968
Cerr << "skip INPROGRESS" << Endl;
7069
continue;
7170
}
@@ -122,7 +121,7 @@ Y_UNIT_TEST_SUITE(TTxDataShardBuildIndexScan) {
122121

123122
auto snapshot = CreateVolatileSnapshot(server, { "/Root/table-1" });
124123

125-
DoBuildIndex(server, sender, "/Root/table-1", "/Root/table-2", snapshot, NKikimrTxDataShard::TEvBuildIndexProgressResponse::DONE);
124+
DoBuildIndex(server, sender, "/Root/table-1", "/Root/table-2", snapshot, NKikimrIndexBuilder::EBuildStatus::DONE);
126125

127126
// Writes to shadow data should not be visible yet
128127
auto data = ReadShardedTable(server, "/Root/table-2");
@@ -174,7 +173,7 @@ Y_UNIT_TEST_SUITE(TTxDataShardBuildIndexScan) {
174173

175174
auto snapshot = CreateVolatileSnapshot(server, { "/Root/table-1" });
176175

177-
DoBuildIndex(server, sender, "/Root/table-1", "/Root/table-2", snapshot, NKikimrTxDataShard::TEvBuildIndexProgressResponse::DONE);
176+
DoBuildIndex(server, sender, "/Root/table-1", "/Root/table-2", snapshot, NKikimrIndexBuilder::EBuildStatus::DONE);
178177

179178
// Writes to shadow data should not be visible yet
180179
auto data = ReadShardedTable(server, "/Root/table-2");

‎ydb/core/tx/datashard/datashard_ut_sample_k.cpp

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,15 @@
66
#include <ydb/core/tx/schemeshard/schemeshard.h>
77
#include <ydb/core/tx/tx_proxy/proxy.h>
88
#include <ydb/core/tx/tx_proxy/upload_rows.h>
9+
#include <ydb/core/protos/index_builder.pb.h>
910

1011
#include <ydb/library/yql/public/issue/yql_issue_message.h>
1112

1213
#include <library/cpp/testing/unittest/registar.h>
1314

1415
template <>
15-
inline void Out<NKikimrTxDataShard::TEvSampleKResponse::EStatus>(IOutputStream& o, NKikimrTxDataShard::TEvSampleKResponse::EStatus status)
16-
{
17-
o << NKikimrTxDataShard::TEvSampleKResponse::EStatus_Name(status);
16+
inline void Out<NKikimrIndexBuilder::EBuildStatus>(IOutputStream& o, NKikimrIndexBuilder::EBuildStatus status) {
17+
o << NKikimrIndexBuilder::EBuildStatus_Name(status);
1818
}
1919

2020
namespace NKikimr {
@@ -71,7 +71,7 @@ Y_UNIT_TEST_SUITE (TTxDataShardSampleKScan) {
7171

7272
TAutoPtr<IEventHandle> handle;
7373
auto reply = runtime.GrabEdgeEventRethrow<TEvDataShard::TEvSampleKResponse>(handle);
74-
UNIT_ASSERT_VALUES_EQUAL(reply->Record.GetStatus(), NKikimrTxDataShard::TEvSampleKResponse::BAD_REQUEST);
74+
UNIT_ASSERT_VALUES_EQUAL(reply->Record.GetStatus(), NKikimrIndexBuilder::EBuildStatus::BAD_REQUEST);
7575
}
7676
}
7777

@@ -115,7 +115,7 @@ Y_UNIT_TEST_SUITE (TTxDataShardSampleKScan) {
115115

116116
TAutoPtr<IEventHandle> handle;
117117
auto reply = runtime.GrabEdgeEventRethrow<TEvDataShard::TEvSampleKResponse>(handle);
118-
UNIT_ASSERT_VALUES_EQUAL(reply->Record.GetStatus(), NKikimrTxDataShard::TEvSampleKResponse::DONE);
118+
UNIT_ASSERT_VALUES_EQUAL(reply->Record.GetStatus(), NKikimrIndexBuilder::EBuildStatus::DONE);
119119

120120
const auto& rows = reply->Record.GetRows();
121121
UNIT_ASSERT(!rows.empty());

‎ydb/core/tx/datashard/sample_k.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ class TSampleKScan final: public TActor<TSampleKScan>, public NTable::IScan {
130130
if (abort == EAbort::None) {
131131
FillResponse();
132132
} else {
133-
Response->Record.SetStatus(NKikimrTxDataShard::TEvSampleKResponse::ABORTED);
133+
Response->Record.SetStatus(NKikimrIndexBuilder::EBuildStatus::ABORTED);
134134
}
135135
LOG_T("Finish " << Debug());
136136
TActivationContext::AsActorContext().MakeFor(SelfId()).Send(ResponseActorId, Response.Release());
@@ -181,7 +181,7 @@ class TSampleKScan final: public TActor<TSampleKScan>, public NTable::IScan {
181181
Response->Record.AddProbabilities(p);
182182
Response->Record.AddRows(std::move(DataRows[i]));
183183
}
184-
Response->Record.SetStatus(NKikimrTxDataShard::TEvSampleKResponse::DONE);
184+
Response->Record.SetStatus(NKikimrIndexBuilder::EBuildStatus::DONE);
185185
}
186186
};
187187

@@ -231,7 +231,7 @@ void TDataShard::HandleSafe(TEvDataShard::TEvSampleKRequest::TPtr& ev, const TAc
231231
response->Record.SetRequestSeqNoRound(seqNo.Round);
232232

233233
auto badRequest = [&](const TString& error) {
234-
response->Record.SetStatus(NKikimrTxDataShard::TEvSampleKResponse::BAD_REQUEST);
234+
response->Record.SetStatus(NKikimrIndexBuilder::EBuildStatus::BAD_REQUEST);
235235
auto issue = response->Record.AddIssues();
236236
issue->set_severity(NYql::TSeverityIds::S_ERROR);
237237
issue->set_message(error);

0 commit comments

Comments
 (0)