Merge branch 'stable-25-1' into changelog/stable-25-1-2025-06-08

Author: galnat

.github/workflows/docs_assign_review.yaml

@@ -0,0 +1,89 @@
+name: Assign a reviewer for documentation
+
+on:
+  pull_request_target:
+    types: [opened]
+
+jobs:
+  assign-docs-reviewer:
+    runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.body, '* Documentation') && !contains(github.event.pull_request.body, '* New feature')
+    steps:
+      - name: Assign a random documentation reviewer, excluding the PR author and busy members
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.YDBOT_TOKEN }}
+          script: |
+            const teamSlug = "primary-docs-reviewers";
+            const org = "ydb-platform";
+            
+            // Get team members
+            const teamMembers = await github.paginate("GET /orgs/{org}/teams/{team_slug}/members", {
+              org,
+              team_slug: teamSlug
+            });
+            if (teamMembers.length === 0) {
+              core.setFailed("No team members found in the team.");
+            }
+            
+            // Get the PR author
+            const prAuthor = context.payload.pull_request.user.login;
+            
+            // Function to check if a user is busy:
+            // It checks for "busy" in their status message OR if they have limitedAvailability set (checkbox).
+            async function isBusy(login) {
+              const query = `
+                query($login: String!) {
+                  user(login: $login) {
+                    status {
+                      message
+                      indicatesLimitedAvailability
+                    }
+                  }
+                }
+              `;
+              try {
+                const result = await github.graphql(query, { login });
+                const status = result.user.status;
+                if (!status) return false;
+                // Consider user busy if limitedAvailability is true or message contains "busy"
+                return status.indicatesLimitedAvailability === true ||
+                       (status.message && status.message.toLowerCase().includes("busy"));
+              } catch (error) {
+                console.log(`Error checking status for ${login}: ${error}`);
+                return false; // On error, assume not busy
+              }
+            }
+            
+            // Filter out the PR author and busy members
+            const availableMembers = [];
+            for (const member of teamMembers) {
+              if (member.login === prAuthor) {
+                console.log(`Skipping the author ${member.login}.`);
+                continue;
+              }
+              if (await isBusy(member.login)) {
+                console.log(`Skipping ${member.login} as they appear busy.`);
+                continue;
+              }
+              availableMembers.push(member);
+            }
+            
+            if (availableMembers.length === 0) {
+              core.setFailed("No available team members after excluding PR author and busy members.");
+            }
+            
+            // Pick a random available member
+            const randomIndex = Math.floor(Math.random() * availableMembers.length);
+            const randomMember = availableMembers[randomIndex].login;
+            
+            // Get the PR number from the event context
+            const prNumber = context.payload.pull_request.number;
+            // Assign the PR to the selected team member
+            await github.rest.issues.addAssignees({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+              assignees: [randomMember]
+            });
+            console.log(`Assigned PR #${prNumber} to ${randomMember}`);

CHANGELOG.md

@@ -3,6 +3,10 @@
 ### Functionality
 
 * 17114:Improved audit logging for user management operations. The audit logs now include details about user modification actions such as password changes, user blocking, and unblocking, making it easier to troubleshoot login issues. [#17114](https://github.com/ydb-platform/ydb/pull/17114) ([flown4qqqq](https://github.com/flown4qqqq))
+* 17691:New UI handler for fetching data from topics. [#17691](https://github.com/ydb-platform/ydb/pull/17691) ([FloatingCrowbar](https://github.com/FloatingCrowbar))
+* 16688:Added to set [grpc compression](https://github.com/grpc/grpc/blob/master/doc/compression_cookbook.md) at channel level. [#16688](https://github.com/ydb-platform/ydb/pull/16688) ([Vitalii Gridnev](https://github.com/gridnevvvit))
+* 18017:Implemented client balancing of partitions when reading using the Kafka protocol (like Kafka itself). Previously, balancing took place on the server. [#18017](https://github.com/ydb-platform/ydb/pull/18017) ([Nikolay Shestakov](https://github.com/nshestakov))
+* 17734:Added automatic cleanup of temporary tables and directories during export to S3. [#17734](https://github.com/ydb-platform/ydb/pull/17734) ([stanislav_shchetinin](https://github.com/stanislav-shchetinin))
 * 18352:Added database audit logs in console's tablet.[#18352](https://github.com/ydb-platform/ydb/pull/18352) ([flown4qqqq](https://github.com/flown4qqqq))
 * 18298:Limited the creation of ReassignerActor to only one active instance to prevent [SelfHeal](https://ydb.tech/docs/ru/maintenance/manual/selfheal) from overloading BSC. [#18298](https://github.com/ydb-platform/ydb/pull/18298) ([Sergey Belyakov](https://github.com/serbel324))
 * 18294:Changed version format from Year.Major.Minor.Hotfix to Year.Major.Minor.Patch.Hotfix [#18294](https://github.com/ydb-platform/ydb/pull/18294) ([Sergey Belyakov](https://github.com/serbel324))
@@ -12,19 +16,16 @@
 
 ### Bug fixes
 
-* 17313:Fixed CopyTable operation to allow copying tables with all column types present in the source table, regardless of feature flag settings. This resolves an issue where copying tables with certain decimal types would fail after version downgrades. [#17313](https://github.com/ydb-platform/ydb/pull/17313) ([azevaykin](https://github.com/azevaykin))
-* 17122:Fixed an rare issue that caused client applications to hang during commit operations. The problem occurred because the `TEvDeletePartition` message could arrive before the `TEvApproveWriteQuota` message. The batch did not send TEvConsumed and this blocked the queue of write quota requests. [#17122](https://github.com/ydb-platform/ydb/pull/17122) ([Alek5andr-Kotov](https://github.com/Alek5andr-Kotov))
-* 18362:Table auto partitioning: Fixed crash when selecting split key from access samples containing a mix of full key and key prefix operations (e.g. exact/range reads). [#18362](https://github.com/ydb-platform/ydb/pull/18362) ([ijon](https://github.com/ijon))
-* 18301:Optimized memory usage in transactions with a large number of participants by changing the storage and resending mechanism for TEvReadSet messages. [#18302](https://github.com/ydb-platform/ydb/pull/18301) ([Alek5andr-Kotov](https://github.com/Alek5andr-Kotov))
+* 18647:[Fixed](https://github.com/ydb-platform/ydb/pull/18647) an [issue](https://github.com/ydb-platform/ydb/issues/17885) where the index type was incorrectly defaulting to GLOBAL SYNC when UNIQUE was explicitly specified in the query. ([Vasily Gerasimov](https://github.com/UgnineSirdis))
+* 18086:Bug fixes for direct read in topics [#18086](https://github.com/ydb-platform/ydb/pull/18086) ([qyryq](https://github.com/qyryq))
+* 16797:Fixed an issue with topic auto-partitioning when the `max_active_partition` configuration parameter was set via the `ALTER TOPIC` statement. [#16797](https://github.com/ydb-platform/ydb/pull/16797) ([Nikolay Shestakov](https://github.com/nshestakov))
 * 18296:Fixed replication continuing to consume disk space when storage was low, which caused VDisks to become read-only. [#18296](https://github.com/ydb-platform/ydb/pull/18296) ([Sergey Belyakov](https://github.com/serbel324))
 * 18271:Fix replication bug #10650 [#18271](https://github.com/ydb-platform/ydb/pull/18271) ([Alexander Rutkovsky](https://github.com/alexvru))
 * 18231:Fix segfault that could happen while retrying Whiteboard requests. [#18231](https://github.com/ydb-platform/ydb/pull/18231) ([Andrei Rykov](https://github.com/StekPerepolnen))
 * 19402:Make legacy signatures for Digest UDF strict again (follows up f97455e). [#19402](https://github.com/ydb-platform/ydb/pull/19402) ([Igor Munkin](https://github.com/igormunkin))
 * 19350:Drop excess langver argument from `TFunctionTypeInfoBuilder` ctor in mkql_udf_ut.cpp. [#19350](https://github.com/ydb-platform/ydb/pull/19350) ([Igor Munkin](https://github.com/igormunkin))
 * 19114:fixes issue in stream lookup join https://github.com/ydb-platform/ydb/issues/19083 [#19114](https://github.com/ydb-platform/ydb/pull/19114) ([Vitalii Gridnev](https://github.com/gridnevvvit))
-* 18954:cherry-pick c6a21d1f, 896fcc7f, 2dcd562c, 42258a1 and fcc18ade. Relates to #18487.
-
-... [#18954](https://github.com/ydb-platform/ydb/pull/18954) ([Igor Munkin](https://github.com/igormunkin))
+* 18954:cherry-pick c6a21d1f, 896fcc7f, 2dcd562c, 42258a1 and fcc18ade. Relates to #18487. [#18954](https://github.com/ydb-platform/ydb/pull/18954) ([Igor Munkin](https://github.com/igormunkin))
 * 18920:Temporarily tweak `DeclareSignature` for `Datetime::Format` and several typeaware UDFs for the incremental upgrade. [#18920](https://github.com/ydb-platform/ydb/pull/18920) ([Igor Munkin](https://github.com/igormunkin))
 * 18873:Temporarily tweak `DeclareSignature` for `DigestFunctionUdf` class for the incremental upgrade. [#18873](https://github.com/ydb-platform/ydb/pull/18873) ([Igor Munkin](https://github.com/igormunkin))
 * 17839:fixes issue where not all tablets are shown for pers queue group on the tablets tab in diagnostics #15230
@@ -43,11 +44,14 @@ fixes issue with long time loading databases page on certain databases due to ti
 fixes issue with no tablets shown for table index on tablets tab #17103
 fixes issue with Optional<Struct> columns are always shown as NULLs #17226
 fixes potential security issue with CORS headers #17670 [#17839](https://github.com/ydb-platform/ydb/pull/17839) ([Alexey Efimov](https://github.com/adameat))
+* 18938:In the table description columns are returned in the same order as they were specified in CREATE TABLE. [#18938](https://github.com/ydb-platform/ydb/pull/18938) ([Ilnaz Nizametdinov](https://github.com/CyberROFL))
+* 18794:[Fixed](https://github.com/db-platform/adb/pull/18794) a rare [bug](https://github.com/ydb-platform/ydb/issues/18615) with PQ tablet restarts. [#18794](https://github.com/ydb-platform/ydb/pull/18794) ([Alek5andr-Kotov](https://github.com/Alek5andr-Kotov))
 
 ### Performance
 
+* 17712:Introduced Intersect, Add and IsEmpty operations to Roaring UDF. [#17712](https://github.com/ydb-platform/ydb/pull/17712) ([jsjant](https://github.com/jsjant))
+* 17578:Added naive bulk And to Roaring UDF. [#17578](https://github.com/ydb-platform/ydb/pull/17578) ([jsjant](https://github.com/jsjant)
 * 19447:Enhancements to shared threads in the actor system.
 We stabilized dynamic resizing of thread count in pools.
 Implemented instant thread pool upscaling to utilize up to 4 cores under sudden bursts of load (this improvement is particularly noticeable in the IC pool) [#19447](https://github.com/ydb-platform/ydb/pull/19447) ([kruall](https://github.com/kruall))
-* 19445:Improved the actor system structures for intensive multithreaded workloads [#19445](https://github.com/ydb-platform/ydb/pull/19445) ([kruall](https://github.com/kruall))
-
+* 19445:Improved the actor system structures for intensive multithreaded workloads [#19445](https://github.com/ydb-platform/ydb/pull/19445) ([kruall](https://github.com/kruall))

ydb/apps/ydb/CHANGELOG.md

@@ -1,3 +1,4 @@
+* Fix `ydb operation get` not working for running operations.
 * Added `--retries` to `ydb workload <clickbenh|tpch|tpcds> run` command.
 * Added `--partition-size` param to `ydb workload <clickbench/tpcds/tpch> init`.
 * Fixed return code of command `ydb workload * run --check-canonical` for the case when benchmark query results differ from canonical ones.

ydb/core/base/table_index.cpp

@@ -1,6 +1,7 @@
 #include "table_index.h"
 
 #include <ydb/core/base/table_vector_index.h>
+#include <ydb/core/protos/tx_datashard.pb.h>
 
 namespace NKikimr::NTableIndex {
 namespace {
@@ -154,7 +155,11 @@ bool IsCompatibleIndex(NKikimrSchemeOp::EIndexType indexType, const TTableColumn
     }
     tmp.clear();
     tmp.insert(table.Keys.begin(), table.Keys.end());
-    tmp.insert(index.KeyColumns.begin(), index.KeyColumns.end() - (isSecondaryIndex ? 0 : 1));
+    if (isSecondaryIndex) {
+        tmp.insert(index.KeyColumns.begin(), index.KeyColumns.end());
+    } else {
+        // Vector indexes allow to add all columns both to index & data
+    }
     if (const auto* broken = IsContains(index.DataColumns, tmp, true)) {
         explain = TStringBuilder()
                   << "the same column can't be used as key and data column for one index, for example " << *broken;
@@ -185,4 +190,36 @@ bool IsBuildImplTable(std::string_view tableName) {
         || tableName.ends_with(NTableVectorKmeansTreeIndex::BuildSuffix1);
 }
 
+static constexpr TClusterId PostingParentFlag = (1ull << 63ull);
+
+// Note: if cluster id is too big, something is wrong with cluster enumeration 
+void EnsureNoPostingParentFlag(TClusterId parent) {
+    Y_ENSURE((parent & PostingParentFlag) == 0);
+}
+
+TClusterId SetPostingParentFlag(TClusterId parent) {
+    EnsureNoPostingParentFlag(parent);
+    return (parent | PostingParentFlag);
+}
+
+TString ToShortDebugString(const NKikimrTxDataShard::TEvReshuffleKMeansRequest& record) {
+    auto copy = record;
+    TStringBuilder result;
+    // clusters are not human readable and can be large like 100Kb+
+    copy.ClearClusters();
+    result << copy.ShortDebugString();
+    result << " Clusters: " << record.ClustersSize();
+    return result;
+}
+
+TString ToShortDebugString(const NKikimrTxDataShard::TEvSampleKResponse& record) {
+    auto copy = record;
+    TStringBuilder result;
+    // rows are not human readable and can be large like 100Kb+
+    copy.ClearRows();
+    result << copy.ShortDebugString();
+    result << " Rows: " << record.RowsSize();
+    return result;
+}
+
 }

ydb/core/base/table_index.h

@@ -12,6 +12,11 @@
 #include <span>
 #include <string_view>
 
+namespace NKikimrTxDataShard {
+    class TEvReshuffleKMeansRequest;
+    class TEvSampleKResponse;
+}
+
 namespace NKikimr {
 
 inline constexpr const char* SYSTEM_COLUMN_PREFIX = "__ydb_";
@@ -38,9 +43,15 @@ bool IsImplTable(std::string_view tableName);
 bool IsBuildImplTable(std::string_view tableName);
 
 using TClusterId = ui64;
-
 inline constexpr auto ClusterIdType = Ydb::Type::UINT64;
 inline constexpr const char* ClusterIdTypeName = "Uint64";
 
+void EnsureNoPostingParentFlag(TClusterId parent);
+
+TClusterId SetPostingParentFlag(TClusterId parent);
+
+TString ToShortDebugString(const NKikimrTxDataShard::TEvReshuffleKMeansRequest& record);
+TString ToShortDebugString(const NKikimrTxDataShard::TEvSampleKResponse& record);
+
 }
 }

ydb/core/driver_lib/run/run.cpp

@@ -519,10 +519,14 @@ static TString ReadFile(const TString& fileName) {
 
 void TKikimrRunner::InitializeGracefulShutdown(const TKikimrRunConfig& runConfig) {
     GracefulShutdownSupported = true;
+    DrainTimeout = TDuration::Seconds(30);
     const auto& config = runConfig.AppConfig.GetShutdownConfig();
     if (config.HasMinDelayBeforeShutdownSeconds()) {
         MinDelayBeforeShutdown = TDuration::Seconds(config.GetMinDelayBeforeShutdownSeconds());
     }
+    if (config.HasDrainTimeoutSeconds()) {
+        DrainTimeout = TDuration::Seconds(config.GetDrainTimeoutSeconds());
+    }
 }
 
 void TKikimrRunner::InitializeKqpController(const TKikimrRunConfig& runConfig) {
@@ -1875,7 +1879,8 @@ void TKikimrRunner::KikimrStop(bool graceful) {
     DisableActorCallstack();
 
     if (drainProgress) {
-        for (ui32 i = 0; i < 300; i++) {
+        ui32 maxTicks = DrainTimeout.MilliSeconds() / 100;
+        for (ui32 i = 0; i < maxTicks; i++) {
             auto cnt = drainProgress->GetOnlineTabletsEstimate();
             if (cnt > 0) {
                 Cerr << "Waiting for drain to complete: " << cnt << " tablets are online on node." << Endl;

ydb/core/driver_lib/run/run.h

@@ -44,6 +44,7 @@ class TKikimrRunner : public virtual TThrRefBase, private IGlobalObjectStorage {
     bool EnabledGrpcService = false;
     bool GracefulShutdownSupported = false;
     TDuration MinDelayBeforeShutdown;
+    TDuration DrainTimeout;
     THolder<NSQS::TAsyncHttpServer> SqsHttp;
 
     THolder<NYdb::TDriver> YdbDriver;

ydb/core/grpc_services/operation_helpers.cpp

@@ -59,6 +59,15 @@ Ydb::TOperationId ToOperationId(const NKikimrIndexBuilder::TIndexBuild& build) {
 void ToOperation(const NKikimrIndexBuilder::TIndexBuild& build, Ydb::Operations::Operation* operation) {
     operation->set_id(NOperationId::ProtoToString(ToOperationId(build)));
     operation->mutable_issues()->CopyFrom(build.GetIssues());
+    if (build.HasStartTime()) {
+        *operation->mutable_create_time() = build.GetStartTime();
+    }
+    if (build.HasEndTime()) {
+        *operation->mutable_end_time() = build.GetEndTime();
+    }
+    if (build.HasUserSID()) {
+        operation->set_created_by(build.GetUserSID());
+    }
 
     switch (build.GetState()) {
         case Ydb::Table::IndexBuildState::STATE_DONE:

ydb/core/grpc_services/rpc_alter_table.cpp

@@ -342,6 +342,9 @@ class TAlterTableRPC : public TRpcSchemeRequestActor<TAlterTableRPC, TEvAlterTab
     void SendAddIndexOpToSS(const TActorContext& ctx, ui64 schemeShardId) {
         SetSchemeShardId(schemeShardId);
         auto ev = std::make_unique<NSchemeShard::TEvIndexBuilder::TEvCreateRequest>(TxId, DatabaseName, std::move(IndexBuildSettings));
+        if (UserToken) {
+            ev->Record.SetUserSID(UserToken->GetUserSID());
+        }
         ForwardToSchemeShard(ctx, std::move(ev));
     }
 

ydb/core/kqp/executer_actor/kqp_data_executer.cpp

@@ -2028,7 +2028,7 @@ class TKqpDataExecuter : public TKqpExecuterBase<TKqpDataExecuter, EExecType::Da
                 auto& stageInfo = TasksGraph.GetStageInfo(stageId);
                 AFL_ENSURE(stageInfo.Id == stageId);
 
-                if (stageInfo.Meta.ShardKind == NSchemeCache::TSchemeCacheRequest::KindAsyncIndexTable) {
+                if (stageInfo.Meta.ShardKind == NSchemeCache::ETableKind::KindAsyncIndexTable) {
                     TMaybe<TString> error;
 
                     if (stageInfo.Meta.ShardKey->RowOperation != TKeyDesc::ERowOperation::Read) {