Skip to content

Commit 090ea14

Browse files
azevaykinazevaykin
authored
25-1: Vector index (#19504)
2 parents 3c434c7 + e98b559 commit 090ea14

File tree

120 files changed

+9982
-5848
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

120 files changed

+9982
-5848
lines changed

‎ydb/apps/ydb/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
* Fix `ydb operation get` not working for running operations.
12
* Added `--retries` to `ydb workload <clickbenh|tpch|tpcds> run` command.
23
* Added `--partition-size` param to `ydb workload <clickbench/tpcds/tpch> init`.
34
* Fixed return code of command `ydb workload * run --check-canonical` for the case when benchmark query results differ from canonical ones.

‎ydb/core/base/table_index.cpp

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "table_index.h"
22

33
#include <ydb/core/base/table_vector_index.h>
4+
#include <ydb/core/protos/tx_datashard.pb.h>
45

56
namespace NKikimr::NTableIndex {
67
namespace {
@@ -154,7 +155,11 @@ bool IsCompatibleIndex(NKikimrSchemeOp::EIndexType indexType, const TTableColumn
154155
}
155156
tmp.clear();
156157
tmp.insert(table.Keys.begin(), table.Keys.end());
157-
tmp.insert(index.KeyColumns.begin(), index.KeyColumns.end() - (isSecondaryIndex ? 0 : 1));
158+
if (isSecondaryIndex) {
159+
tmp.insert(index.KeyColumns.begin(), index.KeyColumns.end());
160+
} else {
161+
// Vector indexes allow to add all columns both to index & data
162+
}
158163
if (const auto* broken = IsContains(index.DataColumns, tmp, true)) {
159164
explain = TStringBuilder()
160165
<< "the same column can't be used as key and data column for one index, for example " << *broken;
@@ -185,4 +190,36 @@ bool IsBuildImplTable(std::string_view tableName) {
185190
|| tableName.ends_with(NTableVectorKmeansTreeIndex::BuildSuffix1);
186191
}
187192

193+
static constexpr TClusterId PostingParentFlag = (1ull << 63ull);
194+
195+
// Note: if cluster id is too big, something is wrong with cluster enumeration
196+
void EnsureNoPostingParentFlag(TClusterId parent) {
197+
Y_ENSURE((parent & PostingParentFlag) == 0);
198+
}
199+
200+
TClusterId SetPostingParentFlag(TClusterId parent) {
201+
EnsureNoPostingParentFlag(parent);
202+
return (parent | PostingParentFlag);
203+
}
204+
205+
TString ToShortDebugString(const NKikimrTxDataShard::TEvReshuffleKMeansRequest& record) {
206+
auto copy = record;
207+
TStringBuilder result;
208+
// clusters are not human readable and can be large like 100Kb+
209+
copy.ClearClusters();
210+
result << copy.ShortDebugString();
211+
result << " Clusters: " << record.ClustersSize();
212+
return result;
213+
}
214+
215+
TString ToShortDebugString(const NKikimrTxDataShard::TEvSampleKResponse& record) {
216+
auto copy = record;
217+
TStringBuilder result;
218+
// rows are not human readable and can be large like 100Kb+
219+
copy.ClearRows();
220+
result << copy.ShortDebugString();
221+
result << " Rows: " << record.RowsSize();
222+
return result;
223+
}
224+
188225
}

‎ydb/core/base/table_index.h

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@
1212
#include <span>
1313
#include <string_view>
1414

15+
namespace NKikimrTxDataShard {
16+
class TEvReshuffleKMeansRequest;
17+
class TEvSampleKResponse;
18+
}
19+
1520
namespace NKikimr {
1621

1722
inline constexpr const char* SYSTEM_COLUMN_PREFIX = "__ydb_";
@@ -38,9 +43,15 @@ bool IsImplTable(std::string_view tableName);
3843
bool IsBuildImplTable(std::string_view tableName);
3944

4045
using TClusterId = ui64;
41-
4246
inline constexpr auto ClusterIdType = Ydb::Type::UINT64;
4347
inline constexpr const char* ClusterIdTypeName = "Uint64";
4448

49+
void EnsureNoPostingParentFlag(TClusterId parent);
50+
51+
TClusterId SetPostingParentFlag(TClusterId parent);
52+
53+
TString ToShortDebugString(const NKikimrTxDataShard::TEvReshuffleKMeansRequest& record);
54+
TString ToShortDebugString(const NKikimrTxDataShard::TEvSampleKResponse& record);
55+
4556
}
4657
}

‎ydb/core/grpc_services/operation_helpers.cpp

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,15 @@ Ydb::TOperationId ToOperationId(const NKikimrIndexBuilder::TIndexBuild& build) {
5959
void ToOperation(const NKikimrIndexBuilder::TIndexBuild& build, Ydb::Operations::Operation* operation) {
6060
operation->set_id(NOperationId::ProtoToString(ToOperationId(build)));
6161
operation->mutable_issues()->CopyFrom(build.GetIssues());
62+
if (build.HasStartTime()) {
63+
*operation->mutable_create_time() = build.GetStartTime();
64+
}
65+
if (build.HasEndTime()) {
66+
*operation->mutable_end_time() = build.GetEndTime();
67+
}
68+
if (build.HasUserSID()) {
69+
operation->set_created_by(build.GetUserSID());
70+
}
6271

6372
switch (build.GetState()) {
6473
case Ydb::Table::IndexBuildState::STATE_DONE:

‎ydb/core/grpc_services/rpc_alter_table.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,9 @@ class TAlterTableRPC : public TRpcSchemeRequestActor<TAlterTableRPC, TEvAlterTab
342342
void SendAddIndexOpToSS(const TActorContext& ctx, ui64 schemeShardId) {
343343
SetSchemeShardId(schemeShardId);
344344
auto ev = std::make_unique<NSchemeShard::TEvIndexBuilder::TEvCreateRequest>(TxId, DatabaseName, std::move(IndexBuildSettings));
345+
if (UserToken) {
346+
ev->Record.SetUserSID(UserToken->GetUserSID());
347+
}
345348
ForwardToSchemeShard(ctx, std::move(ev));
346349
}
347350

‎ydb/core/kqp/executer_actor/kqp_data_executer.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2028,7 +2028,7 @@ class TKqpDataExecuter : public TKqpExecuterBase<TKqpDataExecuter, EExecType::Da
20282028
auto& stageInfo = TasksGraph.GetStageInfo(stageId);
20292029
AFL_ENSURE(stageInfo.Id == stageId);
20302030

2031-
if (stageInfo.Meta.ShardKind == NSchemeCache::TSchemeCacheRequest::KindAsyncIndexTable) {
2031+
if (stageInfo.Meta.ShardKind == NSchemeCache::ETableKind::KindAsyncIndexTable) {
20322032
TMaybe<TString> error;
20332033

20342034
if (stageInfo.Meta.ShardKey->RowOperation != TKeyDesc::ERowOperation::Read) {

‎ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,9 @@ class TKqpSchemeExecuter : public TActorBootstrapped<TKqpSchemeExecuter> {
697697
const auto& buildOp = schemeOp.GetBuildOperation();
698698
SetSchemeShardId(domainInfo->ExtractSchemeShard());
699699
auto req = std::make_unique<NSchemeShard::TEvIndexBuilder::TEvCreateRequest>(TxId, Database, buildOp);
700+
if (UserToken) {
701+
req->Record.SetUserSID(UserToken->GetUserSID());
702+
}
700703
ForwardToSchemeShard(std::move(req));
701704
}
702705

‎ydb/core/kqp/executer_actor/kqp_tasks_graph.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ struct TStageInfoMeta {
4343

4444
THashSet<TKeyDesc::ERowOperation> ShardOperations;
4545
THolder<TKeyDesc> ShardKey;
46-
NSchemeCache::TSchemeCacheRequest::EKind ShardKind = NSchemeCache::TSchemeCacheRequest::EKind::KindUnknown;
46+
NSchemeCache::ETableKind ShardKind = NSchemeCache::ETableKind::KindUnknown;
4747

4848
const NKqpProto::TKqpPhyStage& GetStage(const size_t idx) const {
4949
auto& txBody = Tx.Body;

‎ydb/core/kqp/gateway/kqp_metadata_loader.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,8 @@ TTableMetadataResult GetTableMetadataResult(const NSchemeCache::TSchemeCacheNavi
166166
}
167167
}
168168

169+
tableMeta->IsIndexImplTable = (entry.TableKind != NSchemeCache::ETableKind::KindRegularTable);
170+
169171
tableMeta->Attributes = entry.Attributes;
170172

171173
if (queryName) {

‎ydb/core/kqp/opt/kqp_opt_kql.cpp

Lines changed: 35 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -945,6 +945,11 @@ TExprNode::TPtr HandleWriteTable(const TKiWriteTable& write, TExprContext& ctx,
945945
return BuildFillTable(write, ctx).Ptr();
946946
}
947947
auto& tableData = GetTableData(tablesData, write.DataSink().Cluster(), write.Table().Value());
948+
if (tableData.Metadata->IsIndexImplTable) {
949+
const TString err = "Writing to index implementation tables is not allowed.";
950+
ctx.AddError(YqlIssue(ctx.GetPosition(write.Pos()), TIssuesIds::KIKIMR_BAD_REQUEST, err));
951+
return nullptr;
952+
}
948953
const bool isSink = NeedSinks(tableData, kqpCtx);
949954

950955
auto inputColumnsSetting = GetSetting(write.Settings().Ref(), "input_columns");
@@ -971,28 +976,39 @@ TExprNode::TPtr HandleWriteTable(const TKiWriteTable& write, TExprContext& ctx,
971976
}
972977
}
973978

974-
TExprBase HandleUpdateTable(const TKiUpdateTable& update, TExprContext& ctx, TKqpOptimizeContext& kqpCtx,
979+
TExprNode::TPtr HandleUpdateTable(const TKiUpdateTable& update, TExprContext& ctx, TKqpOptimizeContext& kqpCtx,
975980
const TKikimrTablesData& tablesData, bool withSystemColumns)
976981
{
977982
Y_UNUSED(kqpCtx);
978983
const auto& tableData = GetTableData(tablesData, update.DataSink().Cluster(), update.Table().Value());
984+
if (tableData.Metadata->IsIndexImplTable) {
985+
const TString err = "Writing to index implementation tables is not allowed.";
986+
ctx.AddError(YqlIssue(ctx.GetPosition(update.Pos()), TIssuesIds::KIKIMR_BAD_REQUEST, err));
987+
return nullptr;
988+
}
979989

980990
if (HasIndexesToWrite(tableData)) {
981-
return BuildUpdateTableWithIndex(update, tableData, withSystemColumns, ctx);
991+
return BuildUpdateTableWithIndex(update, tableData, withSystemColumns, ctx).Ptr();
982992
} else {
983-
return BuildUpdateTable(update, tableData, withSystemColumns, ctx);
993+
return BuildUpdateTable(update, tableData, withSystemColumns, ctx).Ptr();
984994
}
985995
}
986996

987-
TExprBase HandleDeleteTable(const TKiDeleteTable& del, TExprContext& ctx, TKqpOptimizeContext& kqpCtx,
997+
TExprNode::TPtr HandleDeleteTable(const TKiDeleteTable& del, TExprContext& ctx, TKqpOptimizeContext& kqpCtx,
988998
const TKikimrTablesData& tablesData, bool withSystemColumns)
989999
{
9901000
Y_UNUSED(kqpCtx);
9911001
auto& tableData = GetTableData(tablesData, del.DataSink().Cluster(), del.Table().Value());
1002+
if (tableData.Metadata->IsIndexImplTable) {
1003+
const TString err = "Writing to index implementation tables is not allowed.";
1004+
ctx.AddError(YqlIssue(ctx.GetPosition(del.Pos()), TIssuesIds::KIKIMR_BAD_REQUEST, err));
1005+
return nullptr;
1006+
}
1007+
9921008
if (HasIndexesToWrite(tableData)) {
993-
return BuildDeleteTableWithIndex(del, tableData, withSystemColumns, ctx);
1009+
return BuildDeleteTableWithIndex(del, tableData, withSystemColumns, ctx).Ptr();
9941010
} else {
995-
return BuildDeleteTable(del, tableData, withSystemColumns, ctx);
1011+
return BuildDeleteTable(del, tableData, withSystemColumns, ctx).Ptr();
9961012
}
9971013
}
9981014

@@ -1051,20 +1067,28 @@ TMaybe<TKqlQueryList> BuildKqlQuery(TKiDataQueryBlocks dataQueryBlocks, const TK
10511067
TNodeOnNodeOwnedMap effectsMap;
10521068
for (const auto& effect : block.Effects()) {
10531069
if (auto maybeWrite = effect.Maybe<TKiWriteTable>()) {
1054-
auto write = HandleWriteTable(maybeWrite.Cast(), ctx, *kqpCtx, tablesData);
1055-
if (!write) {
1070+
auto writeOp = HandleWriteTable(maybeWrite.Cast(), ctx, *kqpCtx, tablesData);
1071+
if (!writeOp) {
10561072
return {};
10571073
}
10581074

1059-
kqlEffects.push_back(TExprBase(write));
1075+
kqlEffects.push_back(TExprBase(writeOp));
10601076
}
10611077

10621078
if (auto maybeUpdate = effect.Maybe<TKiUpdateTable>()) {
1063-
kqlEffects.push_back(HandleUpdateTable(maybeUpdate.Cast(), ctx, *kqpCtx, tablesData, withSystemColumns));
1079+
auto updateOp = HandleUpdateTable(maybeUpdate.Cast(), ctx, *kqpCtx, tablesData, withSystemColumns);
1080+
if (!updateOp) {
1081+
return {};
1082+
}
1083+
kqlEffects.push_back(TExprBase(updateOp));
10641084
}
10651085

10661086
if (auto maybeDelete = effect.Maybe<TKiDeleteTable>()) {
1067-
kqlEffects.push_back(HandleDeleteTable(maybeDelete.Cast(), ctx, *kqpCtx, tablesData, withSystemColumns));
1087+
auto deleteOp = HandleDeleteTable(maybeDelete.Cast(), ctx, *kqpCtx, tablesData, withSystemColumns);
1088+
if (!deleteOp) {
1089+
return {};
1090+
}
1091+
kqlEffects.push_back(TExprBase(deleteOp));
10681092
}
10691093

10701094
if (TExprNode::TPtr result = HandleExternalWrite(effect, ctx, typesCtx)) {

0 commit comments

Comments
 (0)