Skip to content

Commit 89e8285

Browse files
azevaykinazevaykin
committed
Fix ydb operation get not working for running operations (#17001) (#18222)
1 parent fa37fe5 commit 89e8285

File tree

32 files changed

+350
-97
lines changed

32 files changed

+350
-97
lines changed

‎ydb/apps/ydb/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
* Fix `ydb operation get` not working for running operations.
12
* Added `--retries` to `ydb workload <clickbenh|tpch|tpcds> run` command.
23
* Added `--partition-size` param to `ydb workload <clickbench/tpcds/tpch> init`.
34
* Fixed return code of command `ydb workload * run --check-canonical` for the case when benchmark query results differ from canonical ones.

‎ydb/core/grpc_services/operation_helpers.cpp

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,15 @@ Ydb::TOperationId ToOperationId(const NKikimrIndexBuilder::TIndexBuild& build) {
5959
void ToOperation(const NKikimrIndexBuilder::TIndexBuild& build, Ydb::Operations::Operation* operation) {
6060
operation->set_id(NOperationId::ProtoToString(ToOperationId(build)));
6161
operation->mutable_issues()->CopyFrom(build.GetIssues());
62+
if (build.HasStartTime()) {
63+
*operation->mutable_create_time() = build.GetStartTime();
64+
}
65+
if (build.HasEndTime()) {
66+
*operation->mutable_end_time() = build.GetEndTime();
67+
}
68+
if (build.HasUserSID()) {
69+
operation->set_created_by(build.GetUserSID());
70+
}
6271

6372
switch (build.GetState()) {
6473
case Ydb::Table::IndexBuildState::STATE_DONE:

‎ydb/core/grpc_services/rpc_alter_table.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,9 @@ class TAlterTableRPC : public TRpcSchemeRequestActor<TAlterTableRPC, TEvAlterTab
342342
void SendAddIndexOpToSS(const TActorContext& ctx, ui64 schemeShardId) {
343343
SetSchemeShardId(schemeShardId);
344344
auto ev = std::make_unique<NSchemeShard::TEvIndexBuilder::TEvCreateRequest>(TxId, DatabaseName, std::move(IndexBuildSettings));
345+
if (UserToken) {
346+
ev->Record.SetUserSID(UserToken->GetUserSID());
347+
}
345348
ForwardToSchemeShard(ctx, std::move(ev));
346349
}
347350

‎ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,9 @@ class TKqpSchemeExecuter : public TActorBootstrapped<TKqpSchemeExecuter> {
697697
const auto& buildOp = schemeOp.GetBuildOperation();
698698
SetSchemeShardId(domainInfo->ExtractSchemeShard());
699699
auto req = std::make_unique<NSchemeShard::TEvIndexBuilder::TEvCreateRequest>(TxId, Database, buildOp);
700+
if (UserToken) {
701+
req->Record.SetUserSID(UserToken->GetUserSID());
702+
}
700703
ForwardToSchemeShard(std::move(req));
701704
}
702705

‎ydb/core/kqp/ut/indexes/kqp_indexes_vector_ut.cpp

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include <ydb/core/kqp/gateway/kqp_metadata_loader.h>
77
#include <ydb/core/kqp/host/kqp_host_impl.h>
88

9+
#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/operation/operation.h>
910
#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h>
1011
#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/table/table.h>
1112

@@ -609,6 +610,52 @@ Y_UNIT_TEST_SUITE(KqpVectorIndexes) {
609610
// DoPositiveQueriesVectorIndexOrderByCosine(session);
610611
}
611612

613+
Y_UNIT_TEST(BuildIndexTimesAndUser) {
614+
NKikimrConfig::TAppConfig appConfig;
615+
NKikimrConfig::TFeatureFlags featureFlags;
616+
featureFlags.SetEnableVectorIndex(true);
617+
auto setting = NKikimrKqp::TKqpSetting();
618+
auto serverSettings = TKikimrSettings()
619+
.SetAppConfig(appConfig)
620+
.SetFeatureFlags(featureFlags)
621+
.SetKqpSettings({setting});
622+
623+
TKikimrRunner kikimr(serverSettings);
624+
625+
auto now = TInstant::Now();
626+
627+
auto driver = NYdb::TDriver(NYdb::TDriverConfig()
628+
.SetEndpoint(kikimr.GetEndpoint())
629+
.SetDatabase("/Root")
630+
.SetAuthToken("root@builtin"));
631+
auto db = NYdb::NTable::TTableClient(driver);
632+
auto session = DoCreateTableForVectorIndex(db, false);
633+
{
634+
const TString createIndex(Q_(R"(
635+
ALTER TABLE `/Root/TestTable`
636+
ADD INDEX index
637+
GLOBAL USING vector_kmeans_tree
638+
ON (emb)
639+
WITH (distance=cosine, vector_type="uint8", vector_dimension=2, levels=2, clusters=2);
640+
)"));
641+
642+
auto result = session.ExecuteSchemeQuery(createIndex)
643+
.ExtractValueSync();
644+
645+
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
646+
}
647+
{
648+
NYdb::NOperation::TOperationClient client(kikimr.GetDriver());
649+
auto list = client.List<NYdb::NTable::TBuildIndexOperation>().ExtractValueSync();
650+
UNIT_ASSERT_EQUAL(list.GetList().size(), 1);
651+
auto & op = list.GetList()[0];
652+
UNIT_ASSERT_EQUAL(op.Status().GetStatus(), NYdb::EStatus::SUCCESS);
653+
UNIT_ASSERT(op.CreateTime() >= TInstant::Seconds(now.Seconds()));
654+
UNIT_ASSERT(op.EndTime() >= op.CreateTime());
655+
UNIT_ASSERT_EQUAL(op.CreatedBy(), "root@builtin");
656+
}
657+
}
658+
612659
Y_UNIT_TEST(VectorIndexIsNotUpdatable) {
613660
NKikimrConfig::TFeatureFlags featureFlags;
614661
featureFlags.SetEnableVectorIndex(true);

‎ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp

Lines changed: 86 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -2872,29 +2872,32 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
28722872
}
28732873
}
28742874

2875+
void CreateTestTableWithVectorIndex(TSession& session) {
2876+
TString create_index_query = R"(
2877+
--!syntax_v1
2878+
CREATE TABLE `/Root/TestTable` (
2879+
Key Uint64,
2880+
Embedding String,
2881+
PRIMARY KEY (Key),
2882+
INDEX vector_idx
2883+
GLOBAL USING vector_kmeans_tree
2884+
ON (Embedding)
2885+
WITH (similarity=inner_product, vector_type=float, vector_dimension=1024)
2886+
);
2887+
)";
2888+
auto result = session.ExecuteSchemeQuery(create_index_query).ExtractValueSync();
2889+
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
2890+
}
2891+
28752892
Y_UNIT_TEST(AlterTableAlterVectorIndex) {
28762893
NKikimrConfig::TFeatureFlags featureFlags;
28772894
featureFlags.SetEnableVectorIndex(true);
28782895
auto settings = TKikimrSettings().SetFeatureFlags(featureFlags);
28792896
TKikimrRunner kikimr(settings);
28802897
auto db = kikimr.GetTableClient();
28812898
auto session = db.CreateSession().GetValueSync().GetSession();
2882-
{
2883-
TString create_index_query = R"(
2884-
--!syntax_v1
2885-
CREATE TABLE `/Root/TestTable` (
2886-
Key Uint64,
2887-
Embedding String,
2888-
PRIMARY KEY (Key),
2889-
INDEX vector_idx
2890-
GLOBAL USING vector_kmeans_tree
2891-
ON (Embedding)
2892-
WITH (similarity=cosine, vector_type=bit, vector_dimension=1)
2893-
);
2894-
)";
2895-
auto result = session.ExecuteSchemeQuery(create_index_query).ExtractValueSync();
2896-
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
2897-
}
2899+
2900+
CreateTestTableWithVectorIndex(session);
28982901
{
28992902
auto describe = session.DescribeTable("/Root/TestTable/vector_idx/indexImplPostingTable").GetValueSync();
29002903
UNIT_ASSERT_C(describe.IsSuccess(), describe.GetIssues().ToString());
@@ -3137,24 +3140,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
31373140
TKikimrRunner kikimr(settings);
31383141
auto db = kikimr.GetTableClient();
31393142
auto session = db.CreateSession().GetValueSync().GetSession();
3140-
{
3141-
TString create_index_query = R"(
3142-
--!syntax_v1
3143-
CREATE TABLE `/Root/TestTable` (
3144-
Key Uint64,
3145-
Embedding String,
3146-
PRIMARY KEY (Key),
3147-
INDEX vector_idx
3148-
GLOBAL USING vector_kmeans_tree
3149-
ON (Embedding)
3150-
WITH (similarity=inner_product, vector_type=float, vector_dimension=1024)
3151-
);
3152-
)";
3153-
3154-
auto result = session.ExecuteSchemeQuery(create_index_query).ExtractValueSync();
3155-
3156-
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
3157-
}
3143+
CreateTestTableWithVectorIndex(session);
31583144
{
31593145
auto result = session.DescribeTable("/Root/TestTable").ExtractValueSync();
31603146
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS);
@@ -3355,6 +3341,72 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
33553341
}
33563342
}
33573343

3344+
Y_UNIT_TEST(AlterTableRenameVectorIndex) {
3345+
NKikimrConfig::TFeatureFlags featureFlags;
3346+
featureFlags.SetEnableVectorIndex(true);
3347+
auto settings = TKikimrSettings().SetFeatureFlags(featureFlags);
3348+
TKikimrRunner kikimr(settings);
3349+
auto db = kikimr.GetTableClient();
3350+
auto session = db.CreateSession().GetValueSync().GetSession();
3351+
CreateTestTableWithVectorIndex(session);
3352+
{
3353+
auto status = session.ExecuteSchemeQuery(R"(
3354+
--!syntax_v1
3355+
ALTER TABLE `/Root/TestTable` RENAME INDEX vector_idx TO RenamedIndex;
3356+
)").ExtractValueSync();
3357+
UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString());
3358+
}
3359+
3360+
{
3361+
TDescribeTableResult describe = session.DescribeTable("/Root/TestTable").GetValueSync();
3362+
UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS);
3363+
auto indexDesc = describe.GetTableDescription().GetIndexDescriptions();
3364+
UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 1);
3365+
UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexName(), "RenamedIndex");
3366+
UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexColumns().size(), 1);
3367+
UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetDataColumns().size(), 0);
3368+
}
3369+
{
3370+
auto describeLevelTable = session.DescribeTable("/Root/TestTable/RenamedIndex/indexImplLevelTable").GetValueSync();
3371+
UNIT_ASSERT_C(describeLevelTable.IsSuccess(), describeLevelTable.GetIssues().ToString());
3372+
auto describePostingTable = session.DescribeTable("/Root/TestTable/RenamedIndex/indexImplPostingTable").GetValueSync();
3373+
UNIT_ASSERT_C(describePostingTable.IsSuccess(), describePostingTable.GetIssues().ToString());
3374+
}
3375+
}
3376+
3377+
Y_UNIT_TEST(RenameTableWithVectorIndex) {
3378+
NKikimrConfig::TFeatureFlags featureFlags;
3379+
featureFlags.SetEnableVectorIndex(true);
3380+
auto settings = TKikimrSettings().SetFeatureFlags(featureFlags);
3381+
TKikimrRunner kikimr(settings);
3382+
auto db = kikimr.GetTableClient();
3383+
auto session = db.CreateSession().GetValueSync().GetSession();
3384+
CreateTestTableWithVectorIndex(session);
3385+
{
3386+
auto status = session.ExecuteSchemeQuery(R"(
3387+
--!syntax_v1
3388+
ALTER TABLE `/Root/TestTable` RENAME TO TestTableRenamed;
3389+
)").ExtractValueSync();
3390+
UNIT_ASSERT_VALUES_EQUAL_C(status.GetStatus(), EStatus::SUCCESS, status.GetIssues().ToString());
3391+
}
3392+
3393+
{
3394+
TDescribeTableResult describe = session.DescribeTable("/Root/TestTableRenamed").GetValueSync();
3395+
UNIT_ASSERT_EQUAL(describe.GetStatus(), EStatus::SUCCESS);
3396+
auto indexDesc = describe.GetTableDescription().GetIndexDescriptions();
3397+
UNIT_ASSERT_VALUES_EQUAL(indexDesc.size(), 1);
3398+
UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexName(), "vector_idx");
3399+
UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetIndexColumns().size(), 1);
3400+
UNIT_ASSERT_VALUES_EQUAL(indexDesc.back().GetDataColumns().size(), 0);
3401+
}
3402+
{
3403+
auto describeLevelTable = session.DescribeTable("/Root/TestTableRenamed/vector_idx/indexImplLevelTable").GetValueSync();
3404+
UNIT_ASSERT_C(describeLevelTable.IsSuccess(), describeLevelTable.GetIssues().ToString());
3405+
auto describePostingTable = session.DescribeTable("/Root/TestTableRenamed/vector_idx/indexImplPostingTable").GetValueSync();
3406+
UNIT_ASSERT_C(describePostingTable.IsSuccess(), describePostingTable.GetIssues().ToString());
3407+
}
3408+
}
3409+
33583410
Y_UNIT_TEST(AlterTableWithDecimalColumn) {
33593411
TKikimrRunner kikimr;
33603412
auto db = kikimr.GetTableClient();

‎ydb/core/protos/index_builder.proto

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import "ydb/public/api/protos/ydb_status_codes.proto";
44
import "ydb/public/api/protos/ydb_table.proto";
55
import "ydb/public/api/protos/ydb_value.proto";
66
import "google/protobuf/any.proto";
7+
import "google/protobuf/timestamp.proto";
78

89

910
package NKikimrIndexBuilder;
@@ -51,6 +52,9 @@ message TIndexBuild {
5152
optional Ydb.Table.IndexBuildState.State State = 3;
5253
optional TIndexBuildSettings Settings = 4;
5354
optional float Progress = 5 [default = 0];
55+
optional google.protobuf.Timestamp StartTime = 6;
56+
optional google.protobuf.Timestamp EndTime = 7;
57+
optional string UserSID = 8;
5458
}
5559

5660
message TEvCreateRequest {
@@ -60,6 +64,7 @@ message TEvCreateRequest {
6064
optional TIndexBuildSettings Settings = 4;
6165
// Internal flag is true for system-generated operations and is false for those initiated directly by the user.
6266
optional bool Internal = 5 [default = false];
67+
optional string UserSID = 6;
6368
}
6469

6570
message TEvCreateResponse {

‎ydb/core/tx/schemeshard/schemeshard__init.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3409,7 +3409,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
34093409
}
34103410
}
34113411

3412-
// Read KesusAlters
3412+
// Read KesusAlters
34133413
{
34143414
TKesusAlterRows kesusAlterRows;
34153415
if (!LoadKesusAlters(db, kesusAlterRows)) {

‎ydb/core/tx/schemeshard/schemeshard__operation_move_index.cpp

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -575,19 +575,19 @@ TVector<ISubOperation::TPtr> CreateConsistentMoveIndex(TOperationId nextId, cons
575575
result.push_back(CreateDropTableIndex(NextPartId(nextId, result), indexDropping));
576576
}
577577

578-
for (const auto& items: dstIndexPath.Base()->GetChildren()) {
579-
Y_ABORT_UNLESS(context.SS->PathsById.contains(items.second));
580-
auto implPath = context.SS->PathsById.at(items.second);
578+
for (const auto& [name, pathId]: dstIndexPath.Base()->GetChildren()) {
579+
Y_ABORT_UNLESS(context.SS->PathsById.contains(pathId));
580+
auto implPath = context.SS->PathsById.at(pathId);
581581
if (implPath->Dropped()) {
582582
continue;
583583
}
584584

585-
auto implTable = context.SS->PathsById.at(items.second);
585+
auto implTable = context.SS->PathsById.at(pathId);
586586
Y_ABORT_UNLESS(implTable->IsTable());
587587

588588
auto implTableDropping = TransactionTemplate(dstIndexPath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTable);
589589
auto operation = implTableDropping.MutableDrop();
590-
operation->SetName(items.first);
590+
operation->SetName(name);
591591

592592
result.push_back(CreateDropTable(NextPartId(nextId, result), implTableDropping));
593593
}
@@ -596,14 +596,17 @@ TVector<ISubOperation::TPtr> CreateConsistentMoveIndex(TOperationId nextId, cons
596596

597597
result.push_back(CreateMoveTableIndex(NextPartId(nextId, result), MoveTableIndexTask(srcIndexPath, dstIndexPath)));
598598

599-
TString srcImplTableName = srcIndexPath.Base()->GetChildren().begin()->first;
600-
TPath srcImplTable = srcIndexPath.Child(srcImplTableName);
599+
for(const auto& implTable : srcIndexPath.Base()->GetChildren()) {
600+
TString srcImplTableName = implTable.first;
601+
TPath srcImplTable = srcIndexPath.Child(srcImplTableName);
601602

602-
Y_ABORT_UNLESS(srcImplTable.Base()->PathId == srcIndexPath.Base()->GetChildren().begin()->second);
603+
Y_ABORT_UNLESS(srcImplTable.Base()->PathId == implTable.second);
603604

604-
TPath dstImplTable = dstIndexPath.Child(srcImplTableName);
605+
TPath dstImplTable = dstIndexPath.Child(srcImplTableName);
606+
607+
result.push_back(CreateMoveTable(NextPartId(nextId, result), MoveTableTask(srcImplTable, dstImplTable)));
608+
}
605609

606-
result.push_back(CreateMoveTable(NextPartId(nextId, result), MoveTableTask(srcImplTable, dstImplTable)));
607610
return result;
608611
}
609612

‎ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -93,21 +93,20 @@ TVector<ISubOperation::TPtr> CreateConsistentMoveTable(TOperationId nextId, cons
9393
TPath dstIndexPath = dstPath.Child(name);
9494

9595
Y_ABORT_UNLESS(srcChildPath.Base()->PathId == child.second);
96-
Y_VERIFY_S(srcChildPath.Base()->GetChildren().size() == 1,
97-
srcChildPath.PathString() << " has children " << srcChildPath.Base()->GetChildren().size());
9896

9997
result.push_back(CreateMoveTableIndex(NextPartId(nextId, result), MoveTableIndexTask(srcChildPath, dstIndexPath)));
10098

101-
TString srcImplTableName = srcChildPath.Base()->GetChildren().begin()->first;
102-
TPath srcImplTable = srcChildPath.Child(srcImplTableName);
103-
if (srcImplTable.IsDeleted()) {
104-
continue;
105-
}
106-
Y_ABORT_UNLESS(srcImplTable.Base()->PathId == srcChildPath.Base()->GetChildren().begin()->second);
99+
for (const auto& [implTableName, implTablePathId]: srcChildPath.Base()->GetChildren()) {
100+
TPath srcImplTable = srcChildPath.Child(implTableName);
101+
if (srcImplTable.IsDeleted()) {
102+
continue;
103+
}
104+
Y_ABORT_UNLESS(srcImplTable.Base()->PathId == implTablePathId);
107105

108-
TPath dstImplTable = dstIndexPath.Child(srcImplTableName);
106+
TPath dstImplTable = dstIndexPath.Child(implTableName);
109107

110-
result.push_back(CreateMoveTable(NextPartId(nextId, result), MoveTableTask(srcImplTable, dstImplTable)));
108+
result.push_back(CreateMoveTable(NextPartId(nextId, result), MoveTableTask(srcImplTable, dstImplTable)));
109+
}
111110
}
112111

113112
for (const auto& sequence : sequences) {

0 commit comments

Comments
 (0)