From 814e3778ab327e20502b40e08398877e81e811ef Mon Sep 17 00:00:00 2001 From: Natasha Pirogova Date: Mon, 16 Jun 2025 16:38:40 +0300 Subject: [PATCH 01/29] Release notes 24.4.4.12 (#19113) Co-authored-by: Ilnaz Nizametdinov Co-authored-by: ElenaAfina <144937430+ElenaAfina@users.noreply.github.com> --- ydb/docs/en/core/changelog-server.md | 20 +++++++++++++++++++ .../downloads/ydb-open-source-database.md | 3 +++ ydb/docs/ru/core/changelog-server.md | 20 +++++++++++++++++++ .../downloads/ydb-open-source-database.md | 3 +++ 4 files changed, 46 insertions(+) diff --git a/ydb/docs/en/core/changelog-server.md b/ydb/docs/en/core/changelog-server.md index 7fc291875a93..2852243c44d8 100644 --- a/ydb/docs/en/core/changelog-server.md +++ b/ydb/docs/en/core/changelog-server.md @@ -2,6 +2,26 @@ ## Version 24.4 {#24-4} +### Version 24.4.4.12 {#24-4-4-12} + +Release date: June 3, 2025. + +## Performance + +* [Limited](https://github.com/ydb-platform/ydb/pull/17755) the number of internal inflight configuration updates. +* [Optimized](https://github.com/ydb-platform/ydb/issues/18289) memory consumption by PQ tablets. +* [Optimized](https://github.com/ydb-platform/ydb/issues/18473) CPU consumption of Scheme shard and reduced query latencies by checking operation count limits before performing tablet split and merge operations. + +## Bug Fixes + +* [Fixed](https://github.com/ydb-platform/ydb/pull/17123) a rare issue of client applications hanging during transaction commit where deleting partition had been done before write quota update. +* [Fixed](https://github.com/ydb-platform/ydb/pull/17312) an error in copying tables with Decimal type, which caused failures when rolling back to a previous version. +* [Fixed](https://github.com/ydb-platform/ydb/pull/17519) an [issue](https://github.com/ydb-platform/ydb/issues/17499) where a commit without confirmation of writing to a topic led to the blocking of the current and subsequent transactions with topics. +* Fixed transaction hanging when working with topics during tablet [restart](https://github.com/ydb-platform/ydb/issues/17843) or [deletion](https://github.com/ydb-platform/ydb/issues/17915). +* [Fixed](https://github.com/ydb-platform/ydb/pull/18114) [issues](https://github.com/ydb-platform/ydb/issues/18071) with reading messages larger than 6Mb via [Kafka API](./reference/kafka-api). +* [Fixed](https://github.com/ydb-platform/ydb/pull/18319) memory leak during writing to the [topic](./concepts/glossary#topic). +* Fixed errors in processing [nullable columns](https://github.com/ydb-platform/ydb/issues/15701) and [columns with UUID type](https://github.com/ydb-platform/ydb/issues/15697) in row tables. + ### Version 24.4.4.2 {#24-4-4-2} Release date: April 15, 2025 diff --git a/ydb/docs/en/core/downloads/ydb-open-source-database.md b/ydb/docs/en/core/downloads/ydb-open-source-database.md index 81a6fca672a6..0e3cb63c567c 100644 --- a/ydb/docs/en/core/downloads/ydb-open-source-database.md +++ b/ydb/docs/en/core/downloads/ydb-open-source-database.md @@ -9,6 +9,7 @@ See also [{#T}](./yandex-enterprise-database.md). #| || Version | Release date | Download | Changelog || || **v24.4** | > | > | > || +|| v.24.4.4.12 | 03.06.25 | [Binary file](https://binaries.ydb.tech/release/24.4.4.12/ydbd-24.4.4.12-linux-amd64.tar.gz) | [See list](../changelog-server.md#24-4-4-12) || || v.24.4.4.2 | 15.04.25 | [Binary file](https://binaries.ydb.tech/release/24.4.4.2/ydbd-24.4.4.2-linux-amd64.tar.gz) | [See list](../changelog-server.md#24-4-4-2) || || **v24.3** | > | > | > || || v.24.3.15.5 | 06.02.25 | [Binary file](https://binaries.ydb.tech/release/24.3.15.5/ydbd-24.3.15.5-linux-amd64.tar.gz) | [See list](../changelog-server.md#24-3-15-5) || @@ -32,6 +33,7 @@ See also [{#T}](./yandex-enterprise-database.md). #| || Version | Release date | Download | Changelog || || **v24.4** | > | > | > || +|| v.24.4.4.12 | 03.06.25 | `cr.yandex/crptqonuodf51kdj7a7d/ydb:24.4.4.12` | [See list](../changelog-server.md#24-4-4-12) || || v.24.4.4.2 | 15.04.25 | `cr.yandex/crptqonuodf51kdj7a7d/ydb:24.4.4.2` | [See list](../changelog-server.md#24-4-4-2) || || **v24.3** | > | > | > || || v.24.3.15.5 | 06.02.25 | `cr.yandex/crptqonuodf51kdj7a7d/ydb:24.3.15.5` | [See list](../changelog-server.md#24-3-15-5) || @@ -55,6 +57,7 @@ See also [{#T}](./yandex-enterprise-database.md). #| || Version | Release date | Link | Changelog || || **v24.4** | > | > | > || +|| v.24.4.4.2 | 03.06.25 | [https://github.com/ydb-platform/ydb/tree/24.4.4.12](https://github.com/ydb-platform/ydb/tree/24.4.4.12) | [See list](../changelog-server.md#24-4-4-12) || || v.24.4.4.2 | 15.04.25 | [https://github.com/ydb-platform/ydb/tree/24.4.4.2](https://github.com/ydb-platform/ydb/tree/24.4.4.2) | [See list](../changelog-server.md#24-4-4-2) || || **v24.3** | > | > | > || || v.24.3.15.5 | 06.02.25 | [https://github.com/ydb-platform/ydb/tree/24.3.15.5](https://github.com/ydb-platform/ydb/tree/24.3.15.5) | [See list](../changelog-server.md#24-3-15-5) || diff --git a/ydb/docs/ru/core/changelog-server.md b/ydb/docs/ru/core/changelog-server.md index e82295617793..68597cf2e63e 100644 --- a/ydb/docs/ru/core/changelog-server.md +++ b/ydb/docs/ru/core/changelog-server.md @@ -2,6 +2,26 @@ ## Версия 24.4 {#24-4} +### Версия 24.4.4.12 {#24-4-4-12} + +Дата выхода: 3 июня 2025. + +#### Производительность + +* [Ограничено](https://github.com/ydb-platform/ydb/pull/17755) количество одновременно обрабатываемых изменений конфигураций. +* [Оптимизировано](https://github.com/ydb-platform/ydb/issues/18289) потребление памяти PQ-таблетками. +* [Оптимизировано](https://github.com/ydb-platform/ydb/issues/18473) потребление CPU таблеткой Scheme shard, что уменьшило задержки ответов на запросы. Теперь лимит на число операций Scheme shard проверяется до выполнения операций разделения и слияния таблеток. + +#### Исправления ошибок + +* [Исправлена](https://github.com/ydb-platform/ydb/pull/17123) редкая ошибка зависания клиентских приложений во время выполнения коммита транзакции, когда удаление партиции совершалось раньше обновления квоты на запись в топик. +* [Исправлена](https://github.com/ydb-platform/ydb/pull/17312) ошибка в копировании таблиц с типом Decimal, которая приводила к сбою при откате на предыдущую версию. +* [Исправлена](https://github.com/ydb-platform/ydb/pull/17519) [ошибка](https://github.com/ydb-platform/ydb/issues/17499), при которой коммит без подтверждения записи в топик приводил к блокировке текущей и следующих транзакций с топиками. +* Исправлены зависания транзакций при работе с топиками при [перезагрузке](https://github.com/ydb-platform/ydb/issues/17843) или [удалении](https://github.com/ydb-platform/ydb/issues/17915) таблетки. +* [Исправлены](https://github.com/ydb-platform/ydb/pull/18114) [проблемы](https://github.com/ydb-platform/ydb/issues/18071) с чтением сообщений больше 6Mb через [Kafka API](./reference/kafka-api). +* [Устранена](https://github.com/ydb-platform/ydb/pull/18319) утечка памяти во время записи в [топик](./concepts/glossary#topic). +* Исправлены ошибки обработки [nullable столбцов](https://github.com/ydb-platform/ydb/issues/15701) и [столбцов с типом UUID]((https://github.com/ydb-platform/ydb/issues/15697)) в строковых таблицах. + ### Версия 24.4.4.2 {#24-4-4-2} Дата выхода: 15 апреля 2025. diff --git a/ydb/docs/ru/core/downloads/ydb-open-source-database.md b/ydb/docs/ru/core/downloads/ydb-open-source-database.md index 9e687617ffd3..7bfd905af716 100644 --- a/ydb/docs/ru/core/downloads/ydb-open-source-database.md +++ b/ydb/docs/ru/core/downloads/ydb-open-source-database.md @@ -9,6 +9,7 @@ #| || Версия | Дата выпуска | Скачать | Список изменений || || **v24.4** | > | > | > || +|| v.24.4.4.12 | 03.06.25 | [Бинарный файл](https://binaries.ydb.tech/release/24.4.4.12/ydbd-24.4.4.12-linux-amd64.tar.gz) | [См. список](../changelog-server.md#24-4-4-12) || || v.24.4.4.2 | 15.04.25 | [Бинарный файл](https://binaries.ydb.tech/release/24.4.4.2/ydbd-24.4.4.2-linux-amd64.tar.gz) | [См. список](../changelog-server.md#24-4-4-2) || || **v24.3** | > | > | > || || v.24.3.15.5 | 06.02.25 | [Бинарный файл](https://binaries.ydb.tech/release/24.3.15.5/ydbd-24.3.15.5-linux-amd64.tar.gz) | [См. список](../changelog-server.md#24-3-15-5) || @@ -32,6 +33,7 @@ #| || Версия | Дата выпуска | Скачать | Список изменений || || **v24.4** | > | > | > || +|| v.24.4.4.12 | 03.06.25 | `cr.yandex/crptqonuodf51kdj7a7d/ydb:24.4.4.12` | [См. список](../changelog-server.md#24-4-4-12) || || v.24.4.4.2 | 15.04.25 | `cr.yandex/crptqonuodf51kdj7a7d/ydb:24.4.4.2` | [См. список](../changelog-server.md#24-4-4-2) || || **v24.3** | > | > | > || || v.24.3.15.5 | 06.02.25 | `cr.yandex/crptqonuodf51kdj7a7d/ydb:24.3.15.5` | [См. список](../changelog-server.md#24-3-15-5) || @@ -55,6 +57,7 @@ #| || Версия | Дата выпуска | Ссылка | Список изменений || || **v24.4** | > | > | > || +|| v.24.4.4.12 | 03.06.25 | [https://github.com/ydb-platform/ydb/tree/24.4.4.12](https://github.com/ydb-platform/ydb/tree/24.4.4.12) | [См. список](../changelog-server.md#24-4-4-12) || || v.24.4.4.2 | 15.04.25 | [https://github.com/ydb-platform/ydb/tree/24.4.4.2](https://github.com/ydb-platform/ydb/tree/24.4.4.2) | [См. список](../changelog-server.md#24-4-4-2) || || **v24.3** | > | > | > || || v.24.3.15.5 | 06.02.25 | [https://github.com/ydb-platform/ydb/tree/24.3.15.5](https://github.com/ydb-platform/ydb/tree/24.3.15.5) | [См. список](../changelog-server.md#24-3-15-5) || From f56ba50375079149ee1868ff6607db63e72c114e Mon Sep 17 00:00:00 2001 From: Nikolay Shestakov Date: Thu, 19 Jun 2025 12:57:26 +0000 Subject: [PATCH 02/29] Fixed cleanup supprotive partition --- ydb/core/persqueue/partition.cpp | 4 ++ .../client/ydb_topic/ut/topic_to_table_ut.cpp | 67 ++++++++++++++++++- .../ut/ut_utils/topic_sdk_test_setup.cpp | 12 ++-- .../ut/ut_utils/topic_sdk_test_setup.h | 6 +- 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 071ae166e728..7b400a893668 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -391,6 +391,10 @@ void TPartition::AddMetaKey(TEvKeyValue::TEvRequest* request) { } bool TPartition::CleanUp(TEvKeyValue::TEvRequest* request, const TActorContext& ctx) { + if (IsSupportive()) { + return false; + } + bool haveChanges = CleanUpBlobs(request, ctx); LOG_TRACE_S(ctx, NKikimrServices::PERSQUEUE, "Have " << request->Record.CmdDeleteRangeSize() << " items to delete old stuff"); diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp b/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp index f4745d6ef8ef..8433e93b4cab 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp @@ -71,7 +71,9 @@ class TFixture : public NUnitTest::TBaseFixture { void CreateTopic(const TString& path = TEST_TOPIC, const TString& consumer = TEST_CONSUMER, size_t partitionCount = 1, - std::optional maxPartitionCount = std::nullopt); + std::optional maxPartitionCount = std::nullopt, + const TDuration retention = TDuration::Hours(1), + bool important = false); TTopicDescription DescribeTopic(const TString& path); void AddConsumer(const TString& topicPath, @@ -388,10 +390,12 @@ void TFixture::WriteMessages(const TVector& messages, void TFixture::CreateTopic(const TString& path, const TString& consumer, size_t partitionCount, - std::optional maxPartitionCount) + std::optional maxPartitionCount, + const TDuration retention, + bool important) { - Setup->CreateTopic(path, consumer, partitionCount, maxPartitionCount); + Setup->CreateTopic(path, consumer, partitionCount, maxPartitionCount, retention, important); } @@ -2696,6 +2700,63 @@ Y_UNIT_TEST_F(The_Transaction_Starts_On_One_Version_And_Ends_On_The_Other, TFixt RestartPQTablet("topic_A", 1); } +Y_UNIT_TEST_F(TestRetentionOnLongTxAndBigMessages, TFixture) +{ + + auto bigMessage = []() { + TStringBuilder sb; + sb.reserve(10_MB); + + union { + size_t i; + char b[sizeof(size_t)]; + } b; + + for (size_t i = 0; i < 10_MB/sizeof(size_t); ++i) { + b.i = RandomNumber(); + sb << b.b; + } + + return sb; + }; + + TString msg = bigMessage(); + + CreateTopic("topic_A", TEST_CONSUMER, 1, 1, TDuration::Seconds(1), true); + + auto session = CreateTableSession(); + auto tx0 = BeginTx(session); + auto tx1 = BeginTx(session); + + WriteToTopic("topic_A", "grp-0", msg, &tx0); + WriteToTopic("topic_A", "grp-1", msg, &tx1); + + Sleep(TDuration::Seconds(3)); + + WriteToTopic("topic_A", "grp-0", "short-message", &tx0); + WriteToTopic("topic_A", "grp-1", "short-message", &tx1); + + WriteToTopic("topic_A", "grp-0", msg, &tx0); + WriteToTopic("topic_A", "grp-1", msg, &tx1); + + Sleep(TDuration::Seconds(3)); + + WriteToTopic("topic_A", "grp-0", msg, &tx0); + WriteToTopic("topic_A", "grp-1", msg, &tx1); + + Sleep(TDuration::Seconds(3)); + + CommitTx(tx0); + CommitTx(tx1); + + //RestartPQTablet("topic_A", 0); + + auto read = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT(read.size() > 0); + UNIT_ASSERT_EQUAL(msg, read[0]); +} + + } } diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.cpp b/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.cpp index dabad5b62f2d..c503009f88d3 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.cpp @@ -22,16 +22,17 @@ TTopicSdkTestSetup::TTopicSdkTestSetup(const TString& testCaseName, const NKikim } } -void TTopicSdkTestSetup::CreateTopicWithAutoscale(const TString& path, const TString& consumer, size_t partitionCount, size_t maxPartitionCount) { +void TTopicSdkTestSetup::CreateTopicWithAutoscale(const std::string& path, const std::string& consumer, size_t partitionCount, size_t maxPartitionCount) { CreateTopic(path, consumer, partitionCount, maxPartitionCount); } -void TTopicSdkTestSetup::CreateTopic(const TString& path, const TString& consumer, size_t partitionCount, std::optional maxPartitionCount) +void TTopicSdkTestSetup::CreateTopic(const std::string& path, const std::string& consumer, size_t partitionCount, std::optional maxPartitionCount, const TDuration retention, bool important) { TTopicClient client(MakeDriver()); TCreateTopicSettings topics; topics + .RetentionPeriod(retention) .BeginConfigurePartitioningSettings() .MinActivePartitions(partitionCount) .MaxActivePartitions(maxPartitionCount.value_or(partitionCount)); @@ -43,13 +44,14 @@ void TTopicSdkTestSetup::CreateTopic(const TString& path, const TString& consume .Strategy(EAutoPartitioningStrategy::ScaleUp); } - TConsumerSettings consumers(topics, consumer); + TConsumerSettings consumers(topics, TString{consumer}); + consumers.Important(important); topics.AppendConsumers(consumers); - auto status = client.CreateTopic(path, topics).GetValueSync(); + auto status = client.CreateTopic(TString{path}, topics).GetValueSync(); UNIT_ASSERT(status.IsSuccess()); - Server.WaitInit(path); + Server.WaitInit(TString{path}); } TTopicDescription TTopicSdkTestSetup::DescribeTopic(const TString& path) diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.h b/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.h index cc459d6bda60..cb0652333e95 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.h +++ b/ydb/public/sdk/cpp/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.h @@ -16,9 +16,9 @@ class TTopicSdkTestSetup { public: TTopicSdkTestSetup(const TString& testCaseName, const NKikimr::Tests::TServerSettings& settings = MakeServerSettings(), bool createTopic = true); - void CreateTopic(const TString& path = TEST_TOPIC, const TString& consumer = TEST_CONSUMER, size_t partitionCount = 1, - std::optional maxPartitionCount = std::nullopt); - void CreateTopicWithAutoscale(const TString& path = TEST_TOPIC, const TString& consumer = TEST_CONSUMER, size_t partitionCount = 1, + void CreateTopic(const std::string& path = TEST_TOPIC, const std::string& consumer = TEST_CONSUMER, size_t partitionCount = 1, + std::optional maxPartitionCount = std::nullopt, const TDuration retention = TDuration::Hours(1), bool important = false); + void CreateTopicWithAutoscale(const std::string& path = TEST_TOPIC, const std::string& consumer = TEST_CONSUMER, size_t partitionCount = 1, size_t maxPartitionCount = 100); TTopicDescription DescribeTopic(const TString& path = TEST_TOPIC); From b3438d4bcf31e553f049f0cff1b201f3a3b7db57 Mon Sep 17 00:00:00 2001 From: Nikolay Shestakov Date: Fri, 20 Jun 2025 05:33:33 +0000 Subject: [PATCH 03/29] fix --- .../sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp b/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp index 8433e93b4cab..8bd28cc1c718 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp @@ -2706,15 +2706,8 @@ Y_UNIT_TEST_F(TestRetentionOnLongTxAndBigMessages, TFixture) auto bigMessage = []() { TStringBuilder sb; sb.reserve(10_MB); - - union { - size_t i; - char b[sizeof(size_t)]; - } b; - - for (size_t i = 0; i < 10_MB/sizeof(size_t); ++i) { - b.i = RandomNumber(); - sb << b.b; + for (size_t i = 0; i < sb.capacity(); ++i) { + sb << RandomNumber(); } return sb; From b1608092cd8e3e82ca0c404cae6a2c7f30bfb50c Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Fri, 20 Jun 2025 14:42:58 +0300 Subject: [PATCH 04/29] [+] service PQ_TX --- ydb/core/persqueue/partition_log.h | 4 + ydb/core/persqueue/pq_impl.cpp | 203 ++++++++++++++-------------- ydb/core/persqueue/transaction.cpp | 18 +-- ydb/library/services/services.proto | 1 + 4 files changed, 115 insertions(+), 111 deletions(-) diff --git a/ydb/core/persqueue/partition_log.h b/ydb/core/persqueue/partition_log.h index a4181119d0b1..49a8c4984364 100644 --- a/ydb/core/persqueue/partition_log.h +++ b/ydb/core/persqueue/partition_log.h @@ -15,4 +15,8 @@ inline TString LogPrefix() { return {}; } #define PQ_LOG_ERROR(stream) LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE, LogPrefix() << stream) #define PQ_LOG_CRIT(stream) LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE, LogPrefix() << stream) +#define PQ_LOG_TX_D(stream) LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::PQ_TX, LogPrefix() << stream) +#define PQ_LOG_TX_I(stream) LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::PQ_TX, LogPrefix() << stream) +#define PQ_LOG_TX_W(stream) LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::PQ_TX, LogPrefix() << stream) + } // namespace NKikimr::NPQ diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp index 6b6516fa4db4..9d274462c98f 100644 --- a/ydb/core/persqueue/pq_impl.cpp +++ b/ydb/core/persqueue/pq_impl.cpp @@ -710,7 +710,7 @@ void TPersQueue::ReadTxInfo(const NKikimrClient::TKeyValueResponse::TReadResult& switch (read.GetStatus()) { case NKikimrProto::OK: { - PQ_LOG_D("has a tx info"); + PQ_LOG_TX_I("has a tx info"); NKikimrPQ::TTabletTxInfo info; Y_ABORT_UNLESS(info.ParseFromString(read.GetValue())); @@ -720,7 +720,7 @@ void TPersQueue::ReadTxInfo(const NKikimrClient::TKeyValueResponse::TReadResult& break; } case NKikimrProto::NODATA: { - PQ_LOG_D("doesn't have tx info"); + PQ_LOG_TX_I("doesn't have tx info"); InitPlanStep(); @@ -731,7 +731,7 @@ void TPersQueue::ReadTxInfo(const NKikimrClient::TKeyValueResponse::TReadResult& return; } - PQ_LOG_D("PlanStep " << PlanStep << ", PlanTxId " << PlanTxId << + PQ_LOG_TX_I("PlanStep " << PlanStep << ", PlanTxId " << PlanTxId << ", ExecStep " << ExecStep << ", ExecTxId " << ExecTxId); } @@ -806,7 +806,7 @@ void TPersQueue::MoveTopTxToCalculating(TDistributedTransaction& tx, const TActorContext& ctx) { std::tie(ExecStep, ExecTxId) = TxQueue.front(); - PQ_LOG_D("New ExecStep " << ExecStep << ", ExecTxId " << ExecTxId); + PQ_LOG_TX_I("New ExecStep " << ExecStep << ", ExecTxId " << ExecTxId); switch (tx.Kind) { case NKikimrPQ::TTransaction::KIND_DATA: @@ -973,14 +973,14 @@ void TPersQueue::ReadConfig(const NKikimrClient::TKeyValueResponse::TReadResult& NKikimrPQ::TTransaction tx; Y_ABORT_UNLESS(tx.ParseFromString(pair.GetValue())); - PQ_LOG_D("Restore Tx. " << + PQ_LOG_TX_I("Restore Tx. " << "TxId: " << tx.GetTxId() << ", Step: " << tx.GetStep() << ", State: " << NKikimrPQ::TTransaction_EState_Name(tx.GetState()) << ", WriteId: " << tx.GetWriteId().ShortDebugString()); if (tx.GetState() == NKikimrPQ::TTransaction::CALCULATED) { - PQ_LOG_D("Fix tx state"); + PQ_LOG_TX_D("Fix tx state"); tx.SetState(NKikimrPQ::TTransaction::PLANNED); } @@ -992,7 +992,7 @@ void TPersQueue::ReadConfig(const NKikimrClient::TKeyValueResponse::TReadResult& } if (tx.HasWriteId()) { - PQ_LOG_D("Link TxId " << tx.GetTxId() << " with WriteId " << GetWriteId(tx)); + PQ_LOG_TX_I("Link TxId " << tx.GetTxId() << " with WriteId " << GetWriteId(tx)); TxWrites[GetWriteId(tx)].TxId = tx.GetTxId(); } } @@ -1626,8 +1626,7 @@ void TPersQueue::ProcessUpdateConfigRequest(TAutoPtr txs; + + for (ui64 txId : GetBindedTxs(tabletId)) { + PQ_LOG_TX_I("Assume tablet " << tabletId << " dead, sending read set acks for tx " << txId); + auto* tx = GetTransaction(ctx, txId); if (!tx) { continue; @@ -3073,7 +3074,7 @@ void TPersQueue::Handle(TEvMediatorTimecast::TEvRegisterTabletResult::TPtr& ev, MediatorTimeCastEntry = message->Entry; Y_ABORT_UNLESS(MediatorTimeCastEntry); - PQ_LOG_D("Registered with mediator time cast"); + PQ_LOG_TX_D("Registered with mediator time cast"); TryWriteTxs(ctx); } @@ -3182,11 +3183,11 @@ void TPersQueue::Handle(TEvPersQueue::TEvCancelTransactionProposal::TPtr& ev, co return; } - PQ_LOG_D("Handle TEvPersQueue::TEvCancelTransactionProposal"); - NKikimrPQ::TEvCancelTransactionProposal& event = ev->Get()->Record; Y_ABORT_UNLESS(event.HasTxId()); + PQ_LOG_TX_W("Handle TEvPersQueue::TEvCancelTransactionProposal for tx " << event.GetTxId()); + if (auto tx = GetTransaction(ctx, event.GetTxId()); tx) { Y_ABORT_UNLESS(tx->State <= NKikimrPQ::TTransaction::PREPARED); @@ -3204,7 +3205,7 @@ void TPersQueue::Handle(TEvPersQueue::TEvProposeTransaction::TPtr& ev, const TAc } const NKikimrPQ::TEvProposeTransaction& event = ev->Get()->GetRecord(); - PQ_LOG_D("Handle TEvPersQueue::TEvProposeTransaction " << event.ShortDebugString()); + PQ_LOG_TX_D("Handle TEvPersQueue::TEvProposeTransaction " << event.ShortDebugString()); switch (event.GetTxBodyCase()) { case NKikimrPQ::TEvProposeTransaction::kData: @@ -3230,7 +3231,7 @@ bool TPersQueue::CheckTxWriteOperation(const NKikimrPQ::TPartitionOperation& ope TPartitionId partitionId(operation.GetPartitionId(), writeId, operation.GetSupportivePartition()); - PQ_LOG_D("PartitionId " << partitionId << " for WriteId " << writeId); + PQ_LOG_TX_D("PartitionId " << partitionId << " for WriteId " << writeId); return Partitions.contains(partitionId); } @@ -3266,7 +3267,7 @@ void TPersQueue::HandleDataTransaction(TAutoPtr partitionId = FindPartitionId(txBody); if (!partitionId.Defined()) { - PQ_LOG_W("TxId " << event.GetTxId() << " unknown partition for WriteId " << txBody.GetWriteId()); + PQ_LOG_TX_W("TxId " << event.GetTxId() << " unknown partition for WriteId " << txBody.GetWriteId()); SendProposeTransactionAbort(ActorIdFromProto(event.GetSourceActor()), event.GetTxId(), NKikimrPQ::TError::INTERNAL, @@ -3334,7 +3335,7 @@ void TPersQueue::HandleDataTransaction(TAutoPtrGet()->Record.ShortDebugString()); + PQ_LOG_TX_D("Handle TEvTxProcessing::TEvPlanStep " << ev->Get()->Record.ShortDebugString()); EvPlanStepQueue.emplace_back(ev->Sender, ev->Release().Release()); @@ -3405,7 +3406,7 @@ void TPersQueue::Handle(TEvTxProcessing::TEvReadSet::TPtr& ev, const TActorConte return; } - PQ_LOG_D("Handle TEvTxProcessing::TEvReadSet " << ev->Get()->Record.ShortDebugString()); + PQ_LOG_TX_D("Handle TEvTxProcessing::TEvReadSet " << ev->Get()->Record.ShortDebugString()); NKikimrTx::TEvReadSet& event = ev->Get()->Record; Y_ABORT_UNLESS(event.HasTxId()); @@ -3415,11 +3416,13 @@ void TPersQueue::Handle(TEvTxProcessing::TEvReadSet::TPtr& ev, const TActorConte ack = std::make_unique(*ev->Get(), TabletID()); } + PQ_LOG_TX_I("Handle TEvTxProcessing::TEvReadSet tx " << event.GetTxId() << " tabletProducer " << event.GetTabletProducer()); + if (auto tx = GetTransaction(ctx, event.GetTxId()); tx && tx->PredicatesReceived.contains(event.GetTabletProducer())) { if ((tx->State > NKikimrPQ::TTransaction::EXECUTED) || ((tx->State == NKikimrPQ::TTransaction::EXECUTED) && !tx->WriteInProgress)) { if (ack) { - PQ_LOG_D("send TEvReadSetAck to " << event.GetTabletProducer()); + PQ_LOG_TX_I("send TEvReadSetAck to " << event.GetTabletProducer() << " tx " << event.GetTxId()); ctx.Send(ev->Sender, ack.release()); return; } @@ -3433,7 +3436,7 @@ void TPersQueue::Handle(TEvTxProcessing::TEvReadSet::TPtr& ev, const TActorConte TryWriteTxs(ctx); } } else if (ack) { - PQ_LOG_D("send TEvReadSetAck to " << event.GetTabletProducer()); + PQ_LOG_TX_I("send TEvReadSetAck to " << event.GetTabletProducer() << " tx " << event.GetTxId()); // // для неизвестных транзакций подтверждение отправляется сразу // @@ -3443,7 +3446,7 @@ void TPersQueue::Handle(TEvTxProcessing::TEvReadSet::TPtr& ev, const TActorConte void TPersQueue::Handle(TEvTxProcessing::TEvReadSetAck::TPtr& ev, const TActorContext& ctx) { - PQ_LOG_D("Handle TEvTxProcessing::TEvReadSetAck " << ev->Get()->Record.ShortDebugString()); + PQ_LOG_TX_I("Handle TEvTxProcessing::TEvReadSetAck " << ev->Get()->Record.ShortDebugString()); NKikimrTx::TEvReadSetAck& event = ev->Get()->Record; Y_ABORT_UNLESS(event.HasTxId()); @@ -3467,7 +3470,7 @@ void TPersQueue::Handle(TEvPQ::TEvTxCalcPredicateResult::TPtr& ev, const TActorC { const TEvPQ::TEvTxCalcPredicateResult& event = *ev->Get(); - PQ_LOG_D("Handle TEvPQ::TEvTxCalcPredicateResult" << + PQ_LOG_TX_D("Handle TEvPQ::TEvTxCalcPredicateResult" << " Step " << event.Step << ", TxId " << event.TxId << ", Partition " << event.Partition << @@ -3491,7 +3494,7 @@ void TPersQueue::Handle(TEvPQ::TEvProposePartitionConfigResult::TPtr& ev, const { const TEvPQ::TEvProposePartitionConfigResult& event = *ev->Get(); - PQ_LOG_D("Handle TEvPQ::TEvProposePartitionConfigResult" << + PQ_LOG_TX_D("Handle TEvPQ::TEvProposePartitionConfigResult" << " Step " << event.Step << ", TxId " << event.TxId << ", Partition " << event.Partition); @@ -3514,7 +3517,7 @@ void TPersQueue::Handle(TEvPQ::TEvTxCommitDone::TPtr& ev, const TActorContext& c { const TEvPQ::TEvTxCommitDone& event = *ev->Get(); - PQ_LOG_D("Handle TEvPQ::TEvTxCommitDone" << + PQ_LOG_TX_I("Handle TEvPQ::TEvTxCommitDone" << " Step " << event.Step << ", TxId " << event.TxId << ", Partition " << event.Partition); @@ -3564,7 +3567,7 @@ bool TPersQueue::CanProcessTxWrites() const void TPersQueue::SubscribeWriteId(const TWriteId& writeId, const TActorContext& ctx) { - PQ_LOG_D("send TEvSubscribeLock for WriteId " << writeId); + PQ_LOG_TX_D("send TEvSubscribeLock for WriteId " << writeId); ctx.Send(NLongTxService::MakeLongTxServiceID(writeId.NodeId), new NLongTxService::TEvLongTxService::TEvSubscribeLock(writeId.KeyId, writeId.NodeId)); } @@ -3572,7 +3575,7 @@ void TPersQueue::SubscribeWriteId(const TWriteId& writeId, void TPersQueue::UnsubscribeWriteId(const TWriteId& writeId, const TActorContext& ctx) { - PQ_LOG_D("send TEvUnsubscribeLock for WriteId " << writeId); + PQ_LOG_TX_D("send TEvUnsubscribeLock for WriteId " << writeId); ctx.Send(NLongTxService::MakeLongTxServiceID(writeId.NodeId), new NLongTxService::TEvLongTxService::TEvUnsubscribeLock(writeId.KeyId, writeId.NodeId)); } @@ -3680,7 +3683,7 @@ void TPersQueue::ProcessProposeTransactionQueue(const TActorContext& ctx) case NKikimrPQ::TTransaction::UNKNOWN: tx.OnProposeTransaction(event, GetAllowedStep(), TabletID()); - PQ_LOG_D("Propose TxId " << tx.TxId << ", WriteId " << tx.WriteId); + PQ_LOG_TX_I("Propose TxId " << tx.TxId << ", WriteId " << tx.WriteId); if (tx.Kind == NKikimrPQ::TTransaction::KIND_CONFIG) { UpdateReadRuleGenerations(tx.TabletConfig); @@ -3692,7 +3695,7 @@ void TPersQueue::ProcessProposeTransactionQueue(const TActorContext& ctx) "PQ %" PRIu64 ", TxId %" PRIu64 ", WriteId {%" PRIu64 ", %" PRIu64 "}", TabletID(), tx.TxId, writeId.NodeId, writeId.KeyId); TTxWriteInfo& writeInfo = TxWrites.at(writeId); - PQ_LOG_D("Link TxId " << tx.TxId << " with WriteId " << writeId); + PQ_LOG_TX_D("Link TxId " << tx.TxId << " with WriteId " << writeId); writeInfo.TxId = tx.TxId; } @@ -3758,17 +3761,13 @@ void TPersQueue::ProcessPlanStepQueue(const TActorContext& ctx) TxQueue.emplace_back(step, txId); SetTxCompleteLagCounter(); } else { - LOG_WARN_S(ctx, NKikimrServices::PERSQUEUE, - "Tablet " << TabletID() << - " Transaction already planned for step " << tx.Step << - ", Step: " << step << - ", TxId: " << txId); + PQ_LOG_TX_W("Transaction already planned for step " << tx.Step << + ", Step: " << step << + ", TxId: " << txId); } } else { - LOG_WARN_S(ctx, NKikimrServices::PERSQUEUE, - "Tablet " << TabletID() << - " Unknown transaction " << txId << - ", Step: " << step); + PQ_LOG_TX_W("Unknown transaction " << txId << + ", Step: " << step); } lastPlannedTxId = txId; @@ -3777,7 +3776,7 @@ void TPersQueue::ProcessPlanStepQueue(const TActorContext& ctx) PlanStep = step; PlanTxId = lastPlannedTxId; - PQ_LOG_D("PlanStep " << PlanStep << ", PlanTxId " << PlanTxId); + PQ_LOG_TX_D("PlanStep " << PlanStep << ", PlanTxId " << PlanTxId); } else { LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, "Tablet " << TabletID() << @@ -3799,7 +3798,7 @@ void TPersQueue::ProcessWriteTxs(const TActorContext& ctx, // There may be cases when in one iteration of a record we change the state of a transaction and delete it auto tx = GetTransaction(ctx, txId); if (tx) { - PQ_LOG_D("write key for TxId " << txId); + PQ_LOG_TX_D("write key for TxId " << txId); tx->AddCmdWrite(request, state); ChangedTxs.emplace(tx->Step, txId); @@ -3815,7 +3814,7 @@ void TPersQueue::ProcessDeleteTxs(const TActorContext& ctx, Y_ABORT_UNLESS(!WriteTxsInProgress, "PQ %" PRIu64, TabletID()); for (ui64 txId : DeleteTxs) { - PQ_LOG_D("delete key for TxId " << txId); + PQ_LOG_TX_D("delete key for TxId " << txId); AddCmdDeleteTx(request, txId); auto tx = GetTransaction(ctx, txId); @@ -3901,7 +3900,7 @@ void TPersQueue::SaveTxWrites(NKikimrPQ::TTabletTxInfo& info) void TPersQueue::ScheduleProposeTransactionResult(const TDistributedTransaction& tx) { - PQ_LOG_D("schedule TEvProposeTransactionResult(PREPARED)"); + PQ_LOG_TX_D("schedule TEvProposeTransactionResult(PREPARED)"); auto event = std::make_unique(); event->Record.SetOrigin(TabletID()); @@ -3946,7 +3945,7 @@ void TPersQueue::SendEvReadSetToReceivers(const TActorContext& ctx, TString body; Y_ABORT_UNLESS(data.SerializeToString(&body)); - PQ_LOG_D("Send TEvTxProcessing::TEvReadSet to " << tx.PredicateRecipients.size() << " receivers. Wait TEvTxProcessing::TEvReadSet from " << tx.PredicatesReceived.size() << " senders."); + PQ_LOG_TX_I("Send TEvTxProcessing::TEvReadSet to " << tx.PredicateRecipients.size() << " receivers. Wait TEvTxProcessing::TEvReadSet from " << tx.PredicatesReceived.size() << " senders."); for (auto& [receiverId, _] : tx.PredicateRecipients) { if (receiverId != TabletID()) { auto event = std::make_unique(tx.Step, @@ -3956,7 +3955,7 @@ void TPersQueue::SendEvReadSetToReceivers(const TActorContext& ctx, TabletID(), body, 0); - PQ_LOG_D("Send TEvReadSet to tablet " << receiverId); + PQ_LOG_TX_I("Send TEvReadSet to tablet " << receiverId << " tx " << tx.TxId); SendToPipe(receiverId, tx, std::move(event), ctx); } } @@ -3965,9 +3964,9 @@ void TPersQueue::SendEvReadSetToReceivers(const TActorContext& ctx, void TPersQueue::SendEvReadSetAckToSenders(const TActorContext& ctx, TDistributedTransaction& tx) { - PQ_LOG_D("TPersQueue::SendEvReadSetAckToSenders"); + PQ_LOG_TX_D("TPersQueue::SendEvReadSetAckToSenders"); for (auto& [target, event] : tx.ReadSetAcks) { - PQ_LOG_D("Send TEvTxProcessing::TEvReadSetAck " << event->ToString()); + PQ_LOG_TX_I("Send TEvTxProcessing::TEvReadSetAck " << event->ToString()); ctx.Send(target, event.release()); } } @@ -3988,13 +3987,13 @@ TMaybe TPersQueue::FindPartitionId(const NKikimrPQ::TDataTransacti if (txBody.HasWriteId() && hasWriteOperation(txBody)) { const TWriteId writeId = GetWriteId(txBody); if (!TxWrites.contains(writeId)) { - PQ_LOG_D("unknown WriteId " << writeId); + PQ_LOG_TX_W("unknown WriteId " << writeId); return Nothing(); } const TTxWriteInfo& writeInfo = TxWrites.at(writeId); if (!writeInfo.Partitions.contains(partitionId)) { - PQ_LOG_D("unknown partition " << partitionId << " for WriteId " << writeId); + PQ_LOG_TX_W("unknown partition " << partitionId << " for WriteId " << writeId); return Nothing(); } @@ -4042,7 +4041,7 @@ void TPersQueue::SendEvTxCalcPredicateToPartitions(const TActorContext& ctx, for (auto& [originalPartitionId, partitionId] : writeInfo.Partitions) { if (!OriginalPartitionExists(originalPartitionId)) { - PQ_LOG_W("Unknown partition " << originalPartitionId << " for TxId " << tx.TxId); + PQ_LOG_TX_W("Unknown partition " << originalPartitionId << " for TxId " << tx.TxId); forceFalse = true; continue; } @@ -4053,7 +4052,7 @@ void TPersQueue::SendEvTxCalcPredicateToPartitions(const TActorContext& ctx, } if (!Partitions.contains(partitionId)) { - PQ_LOG_W("Unknown partition " << partitionId << " for TxId " << tx.TxId); + PQ_LOG_TX_W("Unknown partition " << partitionId << " for TxId " << tx.TxId); forceFalse = true; continue; } @@ -4063,7 +4062,7 @@ void TPersQueue::SendEvTxCalcPredicateToPartitions(const TActorContext& ctx, event->SupportivePartitionActor = partition.Actor; } } else { - PQ_LOG_W("Unknown WriteId " << writeId << " for TxId " << tx.TxId); + PQ_LOG_TX_W("Unknown WriteId " << writeId << " for TxId " << tx.TxId); forceFalse = true; } } @@ -4131,7 +4130,7 @@ void TPersQueue::SendEvProposeTransactionResult(const TActorContext& ctx, result->Record.SetTxId(tx.TxId); result->Record.SetStep(tx.Step); - PQ_LOG_D("TxId: " << tx.TxId << + PQ_LOG_TX_D("TxId: " << tx.TxId << " send TEvPersQueue::TEvProposeTransactionResult(" << NKikimrPQ::TEvProposeTransactionResult_EStatus_Name(result->Record.GetStatus()) << ")"); @@ -4179,7 +4178,7 @@ TDistributedTransaction* TPersQueue::GetTransaction(const TActorContext& ctx, Y_UNUSED(ctx); auto p = Txs.find(txId); if (p == Txs.end()) { - PQ_LOG_W("Unknown transaction " << txId); + PQ_LOG_TX_W("Unknown transaction " << txId); return nullptr; } return &p->second; @@ -4197,7 +4196,7 @@ void TPersQueue::ChangeTxState(TDistributedTransaction& tx, { tx.State = newState; - PQ_LOG_D("TxId " << tx.TxId << + PQ_LOG_TX_I("TxId " << tx.TxId << ", NewState " << NKikimrPQ::TTransaction_EState_Name(tx.State)); } @@ -4230,7 +4229,7 @@ bool TPersQueue::TryChangeTxState(TDistributedTransaction& tx, PushTxInQueue(tx, newState); } - PQ_LOG_D("TxId " << tx.TxId << " moved from " << + PQ_LOG_TX_I("TxId " << tx.TxId << " moved from " << NKikimrPQ::TTransaction_EState_Name(oldState) << " to " << NKikimrPQ::TTransaction_EState_Name(newState)); @@ -4253,7 +4252,7 @@ bool TPersQueue::CanExecute(const TDistributedTransaction& tx) "PQ %" PRIu64 ", TxId %" PRIu64 ", State %s", TabletID(), tx.TxId, NKikimrPQ::TTransaction_EState_Name(tx.State).data()); - PQ_LOG_D("TxId " << tx.TxId << + PQ_LOG_TX_D("TxId " << tx.TxId << " State " << NKikimrPQ::TTransaction_EState_Name(tx.State) << " FrontTxId " << txQueue.front()); @@ -4263,7 +4262,7 @@ bool TPersQueue::CanExecute(const TDistributedTransaction& tx) void TPersQueue::TryExecuteTxs(const TActorContext& ctx, TDistributedTransaction& tx) { - PQ_LOG_D("Try execute txs with state " << NKikimrPQ::TTransaction_EState_Name(tx.State)); + PQ_LOG_TX_D("Try execute txs with state " << NKikimrPQ::TTransaction_EState_Name(tx.State)); TDistributedTransaction::EState oldState = tx.State; @@ -4276,17 +4275,17 @@ void TPersQueue::TryExecuteTxs(const TActorContext& ctx, if (oldState == tx.State) { // The transaction status has not changed. There is no point in watching the transactions behind her. - PQ_LOG_D("TxId " << tx.TxId << " status has not changed"); + PQ_LOG_TX_D("TxId " << tx.TxId << " status has not changed"); return; } auto& txQueue = TxsOrder[oldState]; while (!txQueue.empty()) { - PQ_LOG_D("There are " << txQueue.size() << " txs in the queue " << NKikimrPQ::TTransaction_EState_Name(oldState)); + PQ_LOG_TX_D("There are " << txQueue.size() << " txs in the queue " << NKikimrPQ::TTransaction_EState_Name(oldState)); ui64 txId = txQueue.front(); Y_ABORT_UNLESS(Txs.contains(txId), "unknown TxId %" PRIu64, txId); auto& tx = Txs.at(txId); - PQ_LOG_D("Try execute TxId " << tx.TxId << " Pending " << tx.Pending); + PQ_LOG_TX_D("Try execute TxId " << tx.TxId << " Pending " << tx.Pending); if (!tx.Pending) { // The transaction was not postponed for execution. @@ -4298,7 +4297,7 @@ void TPersQueue::TryExecuteTxs(const TActorContext& ctx, if (oldState == tx.State) { // The transaction status has not changed. There is no point in watching the transactions behind her. - PQ_LOG_D("TxId " << tx.TxId << " status has not changed"); + PQ_LOG_TX_D("TxId " << tx.TxId << " status has not changed"); break; } } @@ -4307,13 +4306,13 @@ void TPersQueue::TryExecuteTxs(const TActorContext& ctx, void TPersQueue::CheckTxState(const TActorContext& ctx, TDistributedTransaction& tx) { - PQ_LOG_D("TxId " << tx.TxId << + PQ_LOG_TX_D("TxId " << tx.TxId << ", State " << NKikimrPQ::TTransaction_EState_Name(tx.State)); if (!CanExecute(tx)) { - PQ_LOG_D("Can't execute TxId " << tx.TxId << " Pending " << tx.Pending); + PQ_LOG_TX_D("Can't execute TxId " << tx.TxId << " Pending " << tx.Pending); tx.Pending = true; - PQ_LOG_D("Wait for TxId " << tx.TxId); + PQ_LOG_TX_D("Wait for TxId " << tx.TxId); return; } @@ -4376,7 +4375,7 @@ void TPersQueue::CheckTxState(const TActorContext& ctx, "PQ %" PRIu64 ", TxId %" PRIu64 ", FrontTxId %" PRIu64, TabletID(), tx.TxId, TxsOrder[tx.State].front()); - PQ_LOG_D("TxQueue.size " << TxQueue.size()); + PQ_LOG_TX_D("TxQueue.size " << TxQueue.size()); MoveTopTxToCalculating(tx, ctx); @@ -4391,7 +4390,7 @@ void TPersQueue::CheckTxState(const TActorContext& ctx, TabletID(), tx.TxId, tx.PartitionRepliesCount, tx.PartitionRepliesExpected); - PQ_LOG_D("Received " << tx.PartitionRepliesCount << + PQ_LOG_TX_D("Received " << tx.PartitionRepliesCount << ", Expected " << tx.PartitionRepliesExpected); if (tx.PartitionRepliesCount == tx.PartitionRepliesExpected) { @@ -4431,7 +4430,7 @@ void TPersQueue::CheckTxState(const TActorContext& ctx, "PQ %" PRIu64 ", TxId %" PRIu64 ", FrontTxId %" PRIu64, TabletID(), tx.TxId, TxsOrder[tx.State].front()); - PQ_LOG_D("HaveParticipantsDecision " << tx.HaveParticipantsDecision()); + PQ_LOG_TX_D("HaveParticipantsDecision " << tx.HaveParticipantsDecision()); if (tx.HaveParticipantsDecision()) { if (tx.GetDecision() == NKikimrTx::TReadSetData::DECISION_COMMIT) { @@ -4460,12 +4459,12 @@ void TPersQueue::CheckTxState(const TActorContext& ctx, TabletID(), tx.TxId, tx.PartitionRepliesCount, tx.PartitionRepliesExpected); - PQ_LOG_D("Received " << tx.PartitionRepliesCount << + PQ_LOG_TX_D("Received " << tx.PartitionRepliesCount << ", Expected " << tx.PartitionRepliesExpected); if (tx.PartitionRepliesCount == tx.PartitionRepliesExpected) { SendEvProposeTransactionResult(ctx, tx); - PQ_LOG_D("complete TxId " << tx.TxId); + PQ_LOG_TX_I("complete TxId " << tx.TxId); switch (tx.Kind) { case NKikimrPQ::TTransaction::KIND_DATA: @@ -4481,7 +4480,7 @@ void TPersQueue::CheckTxState(const TActorContext& ctx, WriteTx(tx, NKikimrPQ::TTransaction::EXECUTED); - PQ_LOG_D("delete partitions for TxId " << tx.TxId); + PQ_LOG_TX_I("delete partitions for TxId " << tx.TxId); BeginDeletePartitions(tx); TryChangeTxState(tx, NKikimrPQ::TTransaction::EXECUTED); @@ -4506,7 +4505,7 @@ void TPersQueue::CheckTxState(const TActorContext& ctx, [[fallthrough]]; case NKikimrPQ::TTransaction::WAIT_RS_ACKS: - PQ_LOG_D("HaveAllRecipientsReceive " << tx.HaveAllRecipientsReceive() << + PQ_LOG_TX_D("HaveAllRecipientsReceive " << tx.HaveAllRecipientsReceive() << ", AllSupportivePartitionsHaveBeenDeleted " << AllSupportivePartitionsHaveBeenDeleted(tx.WriteId)); if (tx.HaveAllRecipientsReceive() && AllSupportivePartitionsHaveBeenDeleted(tx.WriteId)) { DeleteTx(tx); @@ -4518,7 +4517,7 @@ void TPersQueue::CheckTxState(const TActorContext& ctx, case NKikimrPQ::TTransaction::DELETING: // The PQ tablet has persisted its state. Now she can delete the transaction and take the next one. DeleteWriteId(tx.WriteId); - PQ_LOG_D("delete TxId " << tx.TxId); + PQ_LOG_TX_I("delete TxId " << tx.TxId); Txs.erase(tx.TxId); SetTxInFlyCounter(); @@ -4540,7 +4539,7 @@ bool TPersQueue::AllSupportivePartitionsHaveBeenDeleted(const TMaybe& TabletID(), writeId->NodeId, writeId->KeyId); const TTxWriteInfo& writeInfo = TxWrites.at(*writeId); - PQ_LOG_D("WriteId " << *writeId << + PQ_LOG_TX_D("WriteId " << *writeId << " Partitions.size=" << writeInfo.Partitions.size()); bool deleted = writeInfo.Partitions.empty() @@ -4555,7 +4554,7 @@ void TPersQueue::DeleteWriteId(const TMaybe& writeId) return; } - PQ_LOG_D("delete WriteId " << *writeId); + PQ_LOG_TX_I("delete WriteId " << *writeId); TxWrites.erase(*writeId); } @@ -4568,7 +4567,7 @@ void TPersQueue::WriteTx(TDistributedTransaction& tx, NKikimrPQ::TTransaction::E void TPersQueue::DeleteTx(TDistributedTransaction& tx) { - PQ_LOG_D("add an TxId " << tx.TxId << " to the list for deletion"); + PQ_LOG_TX_D("add an TxId " << tx.TxId << " to the list for deletion"); DeleteTxs.insert(tx.TxId); @@ -4676,7 +4675,7 @@ void TPersQueue::SendProposeTransactionResult(const TActorId& target, error->SetReason(reason); } - PQ_LOG_D("TxId: " << txId << + PQ_LOG_TX_I("TxId: " << txId << " send TEvPersQueue::TEvProposeTransactionResult(" << NKikimrPQ::TEvProposeTransactionResult_EStatus_Name(event->Record.GetStatus()) << ")"); @@ -4855,7 +4854,7 @@ void TPersQueue::InitTxsOrder() void TPersQueue::EndInitTransactions() { - PQ_LOG_D("Txs.size=" << Txs.size() << ", PlannedTxs.size=" << PlannedTxs.size()); + PQ_LOG_TX_D("Txs.size=" << Txs.size() << ", PlannedTxs.size=" << PlannedTxs.size()); std::sort(PlannedTxs.begin(), PlannedTxs.end()); for (auto& item : PlannedTxs) { @@ -4863,7 +4862,7 @@ void TPersQueue::EndInitTransactions() } if (!TxQueue.empty()) { - PQ_LOG_D("top tx queue (" << TxQueue.front().first << ", " << TxQueue.front().second << ")"); + PQ_LOG_TX_D("top tx queue (" << TxQueue.front().first << ", " << TxQueue.front().second << ")"); } for (const auto& [_, txId] : TxQueue) { @@ -4875,14 +4874,14 @@ void TPersQueue::EndInitTransactions() Y_ABORT_UNLESS(txId == tx.TxId); if (!TxsOrder.contains(tx.State)) { - PQ_LOG_D("TxsOrder: " << + PQ_LOG_TX_D("TxsOrder: " << txId << " " << NKikimrPQ::TTransaction_EState_Name(tx.State) << " skip"); continue; } PushTxInQueue(tx, tx.State); - PQ_LOG_D("TxsOrder: " << + PQ_LOG_TX_D("TxsOrder: " << txId << " " << NKikimrPQ::TTransaction_EState_Name(tx.State) << " " << tx.Pending); } } @@ -4981,7 +4980,7 @@ void TPersQueue::Handle(TEvPersQueue::TEvProposeTransactionAttach::TPtr &ev, con return; } - PQ_LOG_D("Handle TEvPersQueue::TEvProposeTransactionAttach " << ev->Get()->Record.ShortDebugString()); + PQ_LOG_TX_D("Handle TEvPersQueue::TEvProposeTransactionAttach " << ev->Get()->Record.ShortDebugString()); const ui64 txId = ev->Get()->Record.GetTxId(); NKikimrProto::EReplyStatus status = NKikimrProto::NODATA; @@ -5040,36 +5039,36 @@ void TPersQueue::ProcessCheckPartitionStatusRequests(const TPartitionId& partiti void TPersQueue::Handle(NLongTxService::TEvLongTxService::TEvLockStatus::TPtr& ev) { - PQ_LOG_D("Handle TEvLongTxService::TEvLockStatus " << ev->Get()->Record.ShortDebugString()); + PQ_LOG_TX_D("Handle TEvLongTxService::TEvLockStatus " << ev->Get()->Record.ShortDebugString()); auto& record = ev->Get()->Record; const TWriteId writeId(record.GetLockNode(), record.GetLockId()); if (!TxWrites.contains(writeId)) { // the transaction has already been completed - PQ_LOG_D("unknown WriteId " << writeId); + PQ_LOG_TX_W("unknown WriteId " << writeId); return; } TTxWriteInfo& writeInfo = TxWrites.at(writeId); - PQ_LOG_D("TxWriteInfo: " << + PQ_LOG_TX_D("TxWriteInfo: " << "WriteId " << writeId << ", TxId " << writeInfo.TxId << ", Status " << NKikimrLongTxService::TEvLockStatus_EStatus_Name(writeInfo.LongTxSubscriptionStatus)); writeInfo.LongTxSubscriptionStatus = record.GetStatus(); if (writeInfo.LongTxSubscriptionStatus == NKikimrLongTxService::TEvLockStatus::STATUS_SUBSCRIBED) { - PQ_LOG_D("subscribed WriteId " << writeId); + PQ_LOG_TX_D("subscribed WriteId " << writeId); return; } if (writeInfo.TxId.Defined()) { // the message `TEvProposeTransaction` has already arrived - PQ_LOG_D("there is already a transaction TxId " << writeInfo.TxId << " for WriteId " << writeId); + PQ_LOG_TX_D("there is already a transaction TxId " << writeInfo.TxId << " for WriteId " << writeId); return; } - PQ_LOG_D("delete partitions for WriteId " << writeId); + PQ_LOG_TX_I("delete partitions for WriteId " << writeId << " (longTxService lost tx)"); BeginDeletePartitions(writeInfo); } @@ -5102,7 +5101,7 @@ void TPersQueue::DeletePartition(const TPartitionId& partitionId, const TActorCo void TPersQueue::Handle(TEvPQ::TEvDeletePartitionDone::TPtr& ev, const TActorContext& ctx) { - PQ_LOG_D("Handle TEvPQ::TEvDeletePartitionDone " << ev->Get()->PartitionId); + PQ_LOG_TX_D("Handle TEvPQ::TEvDeletePartitionDone " << ev->Get()->PartitionId); auto* event = ev->Get(); Y_ABORT_UNLESS(event->PartitionId.WriteId.Defined()); @@ -5137,7 +5136,7 @@ void TPersQueue::Handle(TEvPQ::TEvDeletePartitionDone::TPtr& ev, const TActorCon void TPersQueue::Handle(TEvPQ::TEvTransactionCompleted::TPtr& ev, const TActorContext&) { - PQ_LOG_D("Handle TEvPQ::TEvTransactionCompleted" << + PQ_LOG_TX_I("Handle TEvPQ::TEvTransactionCompleted" << " WriteId " << ev->Get()->WriteId); auto* event = ev->Get(); @@ -5158,13 +5157,13 @@ void TPersQueue::Handle(TEvPQ::TEvTransactionCompleted::TPtr& ev, const TActorCo void TPersQueue::BeginDeletePartitions(TTxWriteInfo& writeInfo) { if (writeInfo.Deleting) { - PQ_LOG_D("Already deleting WriteInfo"); + PQ_LOG_TX_D("Already deleting WriteInfo"); return; } for (auto& [_, partitionId] : writeInfo.Partitions) { Y_ABORT_UNLESS(Partitions.contains(partitionId)); const TPartitionInfo& partition = Partitions.at(partitionId); - PQ_LOG_D("send TEvPQ::TEvDeletePartition to partition " << partitionId); + PQ_LOG_TX_D("send TEvPQ::TEvDeletePartition to partition " << partitionId); Send(partition.Actor, new TEvPQ::TEvDeletePartition); } writeInfo.Deleting = true; diff --git a/ydb/core/persqueue/transaction.cpp b/ydb/core/persqueue/transaction.cpp index 5833b30834f8..9bffe296fa67 100644 --- a/ydb/core/persqueue/transaction.cpp +++ b/ydb/core/persqueue/transaction.cpp @@ -213,7 +213,7 @@ void TDistributedTransaction::OnPlanStep(ui64 step) void TDistributedTransaction::OnTxCalcPredicateResult(const TEvPQ::TEvTxCalcPredicateResult& event) { - PQ_LOG_D("Handle TEvTxCalcPredicateResult"); + PQ_LOG_TX_D("Handle TEvTxCalcPredicateResult"); TMaybe decision; @@ -226,7 +226,7 @@ void TDistributedTransaction::OnTxCalcPredicateResult(const TEvPQ::TEvTxCalcPred void TDistributedTransaction::OnProposePartitionConfigResult(const TEvPQ::TEvProposePartitionConfigResult& event) { - PQ_LOG_D("Handle TEvProposePartitionConfigResult"); + PQ_LOG_TX_D("Handle TEvProposePartitionConfigResult"); OnPartitionResult(event, NKikimrTx::TReadSetData::DECISION_COMMIT); @@ -246,14 +246,14 @@ void TDistributedTransaction::OnPartitionResult(const E& event, TMaybe ack) { - PQ_LOG_D("Handle TEvReadSet"); + PQ_LOG_TX_D("Handle TEvReadSet " << TxId); Y_ABORT_UNLESS((Step == Max()) || (event.HasStep() && (Step == event.GetStep()))); Y_ABORT_UNLESS(event.HasTxId() && (TxId == event.GetTxId())); @@ -270,7 +270,7 @@ void TDistributedTransaction::OnReadSet(const NKikimrTx::TEvReadSet& event, p.SetPredicate(data.GetDecision() == NKikimrTx::TReadSetData::DECISION_COMMIT); ++ReadSetCount; - PQ_LOG_D("Predicates " << ReadSetCount << "/" << PredicatesReceived.size()); + PQ_LOG_TX_D("Predicates " << ReadSetCount << "/" << PredicatesReceived.size()); } } else { Y_DEBUG_ABORT("unknown sender tablet %" PRIu64, event.GetTabletProducer()); @@ -279,7 +279,7 @@ void TDistributedTransaction::OnReadSet(const NKikimrTx::TEvReadSet& event, void TDistributedTransaction::OnReadSetAck(const NKikimrTx::TEvReadSetAck& event) { - PQ_LOG_D("Handle TEvReadSetAck"); + PQ_LOG_TX_D("Handle TEvReadSetAck txId " << TxId); Y_ABORT_UNLESS(event.HasStep() && (Step == event.GetStep())); Y_ABORT_UNLESS(event.HasTxId() && (TxId == event.GetTxId())); @@ -293,7 +293,7 @@ void TDistributedTransaction::OnReadSetAck(ui64 tabletId) PredicateRecipients[tabletId] = true; ++PredicateAcksCount; - PQ_LOG_D("Predicate acks " << PredicateAcksCount << "/" << PredicateRecipients.size()); + PQ_LOG_TX_D("Predicate acks " << PredicateAcksCount << "/" << PredicateRecipients.size()); } } @@ -335,7 +335,7 @@ bool TDistributedTransaction::HaveParticipantsDecision() const bool TDistributedTransaction::HaveAllRecipientsReceive() const { - PQ_LOG_D("PredicateAcks: " << PredicateAcksCount << "/" << PredicateRecipients.size()); + PQ_LOG_TX_D("PredicateAcks: " << PredicateAcksCount << "/" << PredicateRecipients.size()); return PredicateRecipients.size() == PredicateAcksCount; } @@ -343,7 +343,7 @@ void TDistributedTransaction::AddCmdWrite(NKikimrClient::TKeyValueRequest& reque EState state) { auto tx = Serialize(state); - PQ_LOG_D("save tx " << tx.ShortDebugString()); + PQ_LOG_TX_D("save tx " << tx.ShortDebugString()); TString value; Y_ABORT_UNLESS(tx.SerializeToString(&value)); diff --git a/ydb/library/services/services.proto b/ydb/library/services/services.proto index 71f8b5bcedae..194700a688de 100644 --- a/ydb/library/services/services.proto +++ b/ydb/library/services/services.proto @@ -185,6 +185,7 @@ enum EServiceKikimr { CHOOSE_PROXY = 444; LB_CONFIG_MANAGER = 445; PQ_PARTITION_CHOOSER = 462; + PQ_TX = 2201; TOKEN_BUILDER = 450; TICKET_PARSER = 455; From 12002b7d33f3c7681e8b83df7263a88ce98b195e Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Sun, 22 Jun 2025 10:13:04 +0300 Subject: [PATCH 05/29] [-] flag tx.WriteInProgress --- ydb/core/persqueue/pq_impl.cpp | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp index 9d274462c98f..a7148ce4f006 100644 --- a/ydb/core/persqueue/pq_impl.cpp +++ b/ydb/core/persqueue/pq_impl.cpp @@ -4194,10 +4194,12 @@ void TPersQueue::PushTxInQueue(TDistributedTransaction& tx, TDistributedTransact void TPersQueue::ChangeTxState(TDistributedTransaction& tx, TDistributedTransaction::EState newState) { - tx.State = newState; + PQ_LOG_TX_I("TxId " << tx.TxId << " moved from " << + NKikimrPQ::TTransaction_EState_Name(tx.State) << + " to " << + NKikimrPQ::TTransaction_EState_Name(newState)); - PQ_LOG_TX_I("TxId " << tx.TxId << - ", NewState " << NKikimrPQ::TTransaction_EState_Name(tx.State)); + tx.State = newState; } bool TPersQueue::TryChangeTxState(TDistributedTransaction& tx, @@ -4229,11 +4231,6 @@ bool TPersQueue::TryChangeTxState(TDistributedTransaction& tx, PushTxInQueue(tx, newState); } - PQ_LOG_TX_I("TxId " << tx.TxId << " moved from " << - NKikimrPQ::TTransaction_EState_Name(oldState) << - " to " << - NKikimrPQ::TTransaction_EState_Name(newState)); - return true; } @@ -4315,6 +4312,12 @@ void TPersQueue::CheckTxState(const TActorContext& ctx, PQ_LOG_TX_D("Wait for TxId " << tx.TxId); return; } + if (tx.WriteInProgress) { + if (tx.State == NKikimrPQ::TTransaction::EXECUTED) { + PQ_LOG_TX_I("You cannot send TEvReadSetAck for TxId: " << tx.TxId << " until the EXECUTED state is saved"); + } + return; + } switch (tx.State) { case NKikimrPQ::TTransaction::UNKNOWN: @@ -4330,12 +4333,10 @@ void TPersQueue::CheckTxState(const TActorContext& ctx, break; case NKikimrPQ::TTransaction::PREPARING: - Y_ABORT_UNLESS(tx.WriteInProgress, + Y_ABORT_UNLESS(!tx.WriteInProgress, "PQ %" PRIu64 ", TxId %" PRIu64, TabletID(), tx.TxId); - tx.WriteInProgress = false; - // scheduled events will be sent to EndWriteTxs ChangeTxState(tx, NKikimrPQ::TTransaction::PREPARED); @@ -4354,12 +4355,10 @@ void TPersQueue::CheckTxState(const TActorContext& ctx, break; case NKikimrPQ::TTransaction::PLANNING: - Y_ABORT_UNLESS(tx.WriteInProgress, + Y_ABORT_UNLESS(!tx.WriteInProgress, "PQ %" PRIu64 ", TxId %" PRIu64, TabletID(), tx.TxId); - tx.WriteInProgress = false; - // scheduled events will be sent to EndWriteTxs TryChangeTxState(tx, NKikimrPQ::TTransaction::PLANNED); @@ -4495,6 +4494,10 @@ void TPersQueue::CheckTxState(const TActorContext& ctx, Y_ABORT_UNLESS(tx.TxId == TxQueue.front().second, "PQ %" PRIu64 ", TxId %" PRIu64 ", FrontTxId %" PRIu64, TabletID(), tx.TxId, TxQueue.front().second); + Y_ABORT_UNLESS(!tx.WriteInProgress, + "PQ %" PRIu64 ", TxId %" PRIu64, + TabletID(), tx.TxId); + TxQueue.pop_front(); SetTxCompleteLagCounter(); @@ -4593,6 +4596,8 @@ void TPersQueue::CheckChangedTxStates(const TActorContext& ctx) "PQ %" PRIu64 ", TxId %" PRIu64, TabletID(), txId); + tx->WriteInProgress = false; + TryExecuteTxs(ctx, *tx); } From fa4c2b8f1690fd973c34b8f0865ed09e70519914 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Tue, 24 Jun 2025 17:45:40 +0300 Subject: [PATCH 06/29] [+] command SendReadSet --- ydb/core/persqueue/pq_impl.h | 1 + ydb/core/persqueue/pq_impl_app.cpp | 4 ++ .../persqueue/pq_impl_app_sendreadset.cpp | 61 +++++++++++++++++++ ydb/core/persqueue/ya.make | 1 + 4 files changed, 67 insertions(+) create mode 100644 ydb/core/persqueue/pq_impl_app_sendreadset.cpp diff --git a/ydb/core/persqueue/pq_impl.h b/ydb/core/persqueue/pq_impl.h index b3e69cc8df6e..c08bd1780c2b 100644 --- a/ydb/core/persqueue/pq_impl.h +++ b/ydb/core/persqueue/pq_impl.h @@ -94,6 +94,7 @@ class TPersQueue : public NKeyValue::TKeyValueFlat { bool OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, const TActorContext& ctx) override; bool OnRenderAppHtmlPageTx(NMon::TEvRemoteHttpInfo::TPtr ev, const TActorContext& ctx); + bool OnSendReadSetToYourself(NMon::TEvRemoteHttpInfo::TPtr& ev, const TActorContext& ctx); void HandleDie(const TActorContext& ctx) override; diff --git a/ydb/core/persqueue/pq_impl_app.cpp b/ydb/core/persqueue/pq_impl_app.cpp index f439a6711104..8677e3562346 100644 --- a/ydb/core/persqueue/pq_impl_app.cpp +++ b/ydb/core/persqueue/pq_impl_app.cpp @@ -228,6 +228,10 @@ bool TPersQueue::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, const TAc return true; } + if (ev->Get()->Cgi().Has("SendReadSet")) { + return OnSendReadSetToYourself(ev, ctx); + } + if (ev->Get()->Cgi().Has("kv")) { return TKeyValueFlat::OnRenderAppHtmlPage(ev, ctx); } diff --git a/ydb/core/persqueue/pq_impl_app_sendreadset.cpp b/ydb/core/persqueue/pq_impl_app_sendreadset.cpp new file mode 100644 index 000000000000..e58c77b47a40 --- /dev/null +++ b/ydb/core/persqueue/pq_impl_app_sendreadset.cpp @@ -0,0 +1,61 @@ +#include "pq_impl.h" + +namespace NKikimr::NPQ { + +TString MakeReadSetData(bool commit) +{ + NKikimrTx::TReadSetData data; + data.SetDecision(commit ? NKikimrTx::TReadSetData::DECISION_COMMIT : NKikimrTx::TReadSetData::DECISION_ABORT); + + TString encoded; + Y_ABORT_UNLESS(data.SerializeToString(&encoded)); + + return encoded; +} + +template +TMaybe GetParameter(NMon::TEvRemoteHttpInfo::TPtr& ev, const TString& name) +{ + if (!ev->Get()->Cgi().Has(name)) { + return Nothing(); + } + return FromString(ev->Get()->Cgi().Get(name)); +} + +bool TPersQueue::OnSendReadSetToYourself(NMon::TEvRemoteHttpInfo::TPtr& ev, const TActorContext& ctx) +{ + auto step = GetParameter(ev, "step"); + auto txId = GetParameter(ev, "txId"); + auto senderTablet = GetParameter(ev, "senderTablet"); + auto decision = GetParameter(ev, "decision"); + + if (!step.Defined() || !txId.Defined() || !senderTablet.Defined() || !decision.Defined()) { + ctx.Send(ev->Sender, new NMon::TEvRemoteJsonInfoRes(R"({"result":"expected step, txId, senderTablet and decision"})")); + return true; + } + + auto* tx = GetTransaction(ctx, *txId); + if (!tx) { + ctx.Send(ev->Sender, new NMon::TEvRemoteJsonInfoRes(R"({"result":"unknown tx"})")); + return true; + } + if (tx->Step != *step) { + ctx.Send(ev->Sender, new NMon::TEvRemoteJsonInfoRes(R"({"result":"invalid step"})")); + return true; + } + + auto event = std::make_unique(*step, + *txId, + *senderTablet, + TabletID(), + *senderTablet, + MakeReadSetData(*decision == "commit"), + 0); + ctx.Send(ctx.SelfID, std::move(event)); + + ctx.Send(ev->Sender, new NMon::TEvRemoteJsonInfoRes(R"({"result":"OK"})")); + + return true; +} + +} diff --git a/ydb/core/persqueue/ya.make b/ydb/core/persqueue/ya.make index 011fc202013a..8db6e400650c 100644 --- a/ydb/core/persqueue/ya.make +++ b/ydb/core/persqueue/ya.make @@ -28,6 +28,7 @@ SRCS( pq.cpp pq_database.cpp pq_impl_app.cpp + pq_impl_app_sendreadset.cpp pq_impl.cpp pq_l2_cache.cpp pq_rl_helpers.cpp From 8a657d85e898b9880e6315d6f9a870175ead9f2e Mon Sep 17 00:00:00 2001 From: Alek5andr-Kotov Date: Wed, 18 Jun 2025 18:20:38 +0300 Subject: [PATCH 07/29] Optimize the writing session creation time (#19807) --- ydb/core/persqueue/writer/writer.cpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/ydb/core/persqueue/writer/writer.cpp b/ydb/core/persqueue/writer/writer.cpp index 466f57b60c91..f7a0d051d040 100644 --- a/ydb/core/persqueue/writer/writer.cpp +++ b/ydb/core/persqueue/writer/writer.cpp @@ -226,6 +226,10 @@ class TPartitionWriter: public TActorBootstrapped, private TRl /// GetWriteId void GetWriteId(const TActorContext& ctx) { + INFO("Start of a request to KQP for a WriteId. " << + "SessionId: " << Opts.SessionId << + " TxId: " << Opts.TxId); + auto ev = MakeWriteIdRequest(); ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); Become(&TThis::StateGetWriteId); @@ -242,6 +246,10 @@ class TPartitionWriter: public TActorBootstrapped, private TRl } void HandleWriteId(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { + INFO("End of the request to KQP for the WriteId. " << + "SessionId: " << Opts.SessionId << + " TxId: " << Opts.TxId); + auto& record = ev->Get()->Record.GetRef(); switch (record.GetYdbStatus()) { case Ydb::StatusIds::SUCCESS: @@ -376,11 +384,19 @@ class TPartitionWriter: public TActorBootstrapped, private TRl Y_ABORT_UNLESS(HasWriteId()); Y_ABORT_UNLESS(HasSupportivePartitionId()); + INFO("Start of a request to KQP to save PartitionId. " << + "SessionId: " << Opts.SessionId << + " TxId: " << Opts.TxId); + auto ev = MakeWriteIdRequest(); ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); } void HandlePartitionIdSaved(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext&) { + INFO("End of a request to KQP to save PartitionId. " << + "SessionId: " << Opts.SessionId << + " TxId: " << Opts.TxId); + auto& record = ev->Get()->Record.GetRef(); switch (record.GetYdbStatus()) { case Ydb::StatusIds::SUCCESS: @@ -929,7 +945,10 @@ class TPartitionWriter: public TActorBootstrapped, private TRl using IRetryState = IRetryPolicy::IRetryState; static IRetryPolicy::TPtr GetRetryPolicy() { - return IRetryPolicy::GetExponentialBackoffPolicy(Retryable); + return IRetryPolicy::GetExponentialBackoffPolicy(Retryable, + TDuration::MilliSeconds(10), + TDuration::MilliSeconds(10), + TDuration::MilliSeconds(100)); }; static ERetryErrorClass Retryable(Ydb::StatusIds::StatusCode code) { From c8274cde9ff62f3235d71934593aaf0d52f25614 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Mon, 23 Jun 2025 14:12:55 +0300 Subject: [PATCH 08/29] [*] unnecessary SchemeCache calls --- .../kqp/session_actor/kqp_session_actor.cpp | 36 +++++++++++++++++-- ydb/core/kqp/topics/kqp_topics.cpp | 6 +++- ydb/core/kqp/topics/kqp_topics.h | 2 ++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index b2d1c49bf9dd..059d80eb4252 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -508,6 +508,21 @@ class TKqpSessionActor : public TActorBootstrapped { CompileQuery(); } + bool AreAllTheTopicsAndPartitionsKnown() const { + const NKikimrKqp::TTopicOperationsRequest& operations = QueryState->GetTopicOperations(); + for (const auto& topic : operations.GetTopics()) { + auto path = CanonizePath(NPersQueue::GetFullTopicPath(TlsActivationContext->AsActorContext(), + QueryState->GetDatabase(), topic.path())); + + for (const auto& partition : topic.partitions()) { + if (!QueryState->TxCtx->TopicOperations.HasThisPartitionAlreadyBeenAdded(path, partition.partition_id())) { + return false; + } + } + } + return true; + } + void AddOffsetsToTransaction() { YQL_ENSURE(QueryState); if (!PrepareQueryTransaction()) { @@ -516,10 +531,25 @@ class TKqpSessionActor : public TActorBootstrapped { QueryState->AddOffsetsToTransaction(); - auto navigate = QueryState->BuildSchemeCacheNavigate(); + if (!AreAllTheTopicsAndPartitionsKnown()) { + auto navigate = QueryState->BuildSchemeCacheNavigate(); + Become(&TKqpSessionActor::ExecuteState); + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigate.release())); + return; + } - Become(&TKqpSessionActor::ExecuteState); - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigate.release())); + TString message; + if (!QueryState->TryMergeTopicOffsets(QueryState->TopicOperations, message)) { + ythrow TRequestFail(Ydb::StatusIds::BAD_REQUEST) << message; + } + + if (HasTopicWriteOperations() && !HasTopicWriteId()) { + Become(&TKqpSessionActor::ExecuteState); + Send(MakeTxProxyID(), new TEvTxUserProxy::TEvAllocateTxId, 0, QueryState->QueryId); + return; + } + + ReplySuccess(); } void CompileQuery() { diff --git a/ydb/core/kqp/topics/kqp_topics.cpp b/ydb/core/kqp/topics/kqp_topics.cpp index 6b90767ff7b7..0b13003ad82a 100644 --- a/ydb/core/kqp/topics/kqp_topics.cpp +++ b/ydb/core/kqp/topics/kqp_topics.cpp @@ -137,7 +137,6 @@ void TTopicPartitionOperations::Merge(const TTopicPartitionOperations& rhs) { Y_ABORT_UNLESS(Topic_.Empty() || Topic_ == rhs.Topic_); Y_ABORT_UNLESS(Partition_.Empty() || Partition_ == rhs.Partition_); - Y_ABORT_UNLESS(TabletId_.Empty() || TabletId_ == rhs.TabletId_); if (Topic_.Empty()) { Topic_ = rhs.Topic_; @@ -356,6 +355,11 @@ bool TTopicOperations::ProcessSchemeCacheNavigate(const NSchemeCache::TSchemeCac return true; } +bool TTopicOperations::HasThisPartitionAlreadyBeenAdded(const TString& topic, ui32 partitionId) const +{ + return Operations_.contains({topic, partitionId}); +} + void TTopicOperations::BuildTopicTxs(TTopicOperationTransactions& txs) { for (auto& [_, operations] : Operations_) { diff --git a/ydb/core/kqp/topics/kqp_topics.h b/ydb/core/kqp/topics/kqp_topics.h index f4ff0bb180e2..200a076d25d4 100644 --- a/ydb/core/kqp/topics/kqp_topics.h +++ b/ydb/core/kqp/topics/kqp_topics.h @@ -125,6 +125,8 @@ class TTopicOperations { size_t GetSize() const; + bool HasThisPartitionAlreadyBeenAdded(const TString& topic, ui32 partitionId) const; + private: THashMap Operations_; bool HasReadOperations_ = false; From a12b16af76d089fb35fe6932567e0aeac603ea7e Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Mon, 23 Jun 2025 14:28:27 +0300 Subject: [PATCH 09/29] [+] logging --- ydb/core/persqueue/writer/writer.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ydb/core/persqueue/writer/writer.cpp b/ydb/core/persqueue/writer/writer.cpp index f7a0d051d040..f4993fb3cbc9 100644 --- a/ydb/core/persqueue/writer/writer.cpp +++ b/ydb/core/persqueue/writer/writer.cpp @@ -195,6 +195,7 @@ class TPartitionWriter: public TActorBootstrapped, private TRl } if (auto delay = RetryState->GetNextRetryDelay(code); delay.Defined()) { + INFO("Repeat the request to KQP in " << *delay); Schedule(*delay, new TEvents::TEvWakeup()); } } @@ -248,7 +249,8 @@ class TPartitionWriter: public TActorBootstrapped, private TRl void HandleWriteId(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { INFO("End of the request to KQP for the WriteId. " << "SessionId: " << Opts.SessionId << - " TxId: " << Opts.TxId); + " TxId: " << Opts.TxId << + " Status: " << ev->Get()->Record.GetRef().GetYdbStatus()); auto& record = ev->Get()->Record.GetRef(); switch (record.GetYdbStatus()) { @@ -395,7 +397,8 @@ class TPartitionWriter: public TActorBootstrapped, private TRl void HandlePartitionIdSaved(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext&) { INFO("End of a request to KQP to save PartitionId. " << "SessionId: " << Opts.SessionId << - " TxId: " << Opts.TxId); + " TxId: " << Opts.TxId << + " Status: " << ev->Get()->Record.GetRef().GetYdbStatus()); auto& record = ev->Get()->Record.GetRef(); switch (record.GetYdbStatus()) { From 3a2960ee9928ed480f1729179c8cfef394c0dbbb Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Tue, 24 Jun 2025 00:07:59 +0300 Subject: [PATCH 10/29] [+] cache for SchemeNavigate --- .../kqp/session_actor/kqp_session_actor.cpp | 22 +++++++++++++ ydb/core/kqp/topics/kqp_topics.cpp | 31 +++++++++++++++++-- ydb/core/kqp/topics/kqp_topics.h | 4 ++- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index 059d80eb4252..742a2c6fbda6 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -222,6 +222,9 @@ class TKqpSessionActor : public TActorBootstrapped { << "ActorId: " << SelfId() << ", " << "ActorState: " << CurrentStateFuncName() << ", "; if (Y_LIKELY(QueryState)) { + if (QueryState->HasTxControl()) { + result << " TxId: " << QueryState->GetTxControl().tx_id() << ", "; + } result << "TraceId: " << QueryState->UserRequestContext->TraceId << ", "; } return result; @@ -524,6 +527,7 @@ class TKqpSessionActor : public TActorBootstrapped { } void AddOffsetsToTransaction() { + LOG_I("begin request for TopicOperations"); YQL_ENSURE(QueryState); if (!PrepareQueryTransaction()) { return; @@ -534,6 +538,7 @@ class TKqpSessionActor : public TActorBootstrapped { if (!AreAllTheTopicsAndPartitionsKnown()) { auto navigate = QueryState->BuildSchemeCacheNavigate(); Become(&TKqpSessionActor::ExecuteState); + LOG_I("begin request for SchemeNavigate"); Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigate.release())); return; } @@ -549,7 +554,10 @@ class TKqpSessionActor : public TActorBootstrapped { return; } + LOG_I("end request for TopicOperations"); ReplySuccess(); + + LOG_I("after ReplySuccess"); } void CompileQuery() { @@ -2490,8 +2498,10 @@ class TKqpSessionActor : public TActorBootstrapped { } void ProcessTopicOps(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + LOG_I("end request for SchemeNavigate"); YQL_ENSURE(ev->Get()->Request); if (ev->Get()->Request->Cookie < QueryId) { + LOG_I("unexpected return #2"); return; } @@ -2517,21 +2527,33 @@ class TKqpSessionActor : public TActorBootstrapped { } if (HasTopicWriteOperations() && !HasTopicWriteId()) { + LOG_I("begin request for WriteId"); Send(MakeTxProxyID(), new TEvTxUserProxy::TEvAllocateTxId, 0, QueryState->QueryId); } else { + LOG_I("end request for TopicOperations"); ReplySuccess(); + + LOG_I("after ReplySuccess"); } } void Handle(TEvTxUserProxy::TEvAllocateTxIdResult::TPtr& ev) { + LOG_I("end request for WriteId"); if (CurrentStateFunc() != &TThis::ExecuteState || ev->Cookie < QueryId) { + LOG_I("unexpected return #1"); return; } YQL_ENSURE(QueryState); YQL_ENSURE(QueryState->GetAction() == NKikimrKqp::QUERY_ACTION_TOPIC); + SetTopicWriteId(NLongTxService::TLockHandle(ev->Get()->TxId, TActivationContext::ActorSystem())); + + LOG_I("end request for TopicOperations"); ReplySuccess(); + + LOG_I("current state: " << CurrentStateFuncName()); + LOG_I("after ReplySuccess"); } bool HasTopicWriteOperations() const { diff --git a/ydb/core/kqp/topics/kqp_topics.cpp b/ydb/core/kqp/topics/kqp_topics.cpp index 0b13003ad82a..f7de2943b9c3 100644 --- a/ydb/core/kqp/topics/kqp_topics.cpp +++ b/ydb/core/kqp/topics/kqp_topics.cpp @@ -339,6 +339,8 @@ bool TTopicOperations::ProcessSchemeCacheNavigate(const NSchemeCache::TSchemeCac p->second.SetTabletId(partition.GetTabletId()); } } + + CachedNavigateResult_[path] = result; } else { builder << "Topic '" << JoinPath(result.Path) << "' is missing"; @@ -355,9 +357,34 @@ bool TTopicOperations::ProcessSchemeCacheNavigate(const NSchemeCache::TSchemeCac return true; } -bool TTopicOperations::HasThisPartitionAlreadyBeenAdded(const TString& topic, ui32 partitionId) const +bool TTopicOperations::HasThisPartitionAlreadyBeenAdded(const TString& topicPath, ui32 partitionId) { - return Operations_.contains({topic, partitionId}); + if (Operations_.contains({topicPath, partitionId})) { + return true; + } + if (!CachedNavigateResult_.contains(topicPath)) { + return false; + } + + const NSchemeCache::TSchemeCacheNavigate::TEntry& entry = + CachedNavigateResult_.at(topicPath); + const NKikimrSchemeOp::TPersQueueGroupDescription& description = + entry.PQGroupInfo->Description; + + TString path = CanonizePath(entry.Path); + Y_ABORT_UNLESS(path == topicPath, + "path=%s, topicPath=%s", + path.data(), topicPath.data()); + + for (const auto& partition : description.GetPartitions()) { + if (partition.GetPartitionId() == partitionId) { + TTopicPartition key{topicPath, partitionId}; + Operations_[key].SetTabletId(partition.GetTabletId()); + return true; + } + } + + return false; } void TTopicOperations::BuildTopicTxs(TTopicOperationTransactions& txs) diff --git a/ydb/core/kqp/topics/kqp_topics.h b/ydb/core/kqp/topics/kqp_topics.h index 200a076d25d4..d8022b0c99de 100644 --- a/ydb/core/kqp/topics/kqp_topics.h +++ b/ydb/core/kqp/topics/kqp_topics.h @@ -125,7 +125,7 @@ class TTopicOperations { size_t GetSize() const; - bool HasThisPartitionAlreadyBeenAdded(const TString& topic, ui32 partitionId) const; + bool HasThisPartitionAlreadyBeenAdded(const TString& topic, ui32 partitionId); private: THashMap Operations_; @@ -134,6 +134,8 @@ class TTopicOperations { TMaybe Consumer_; NLongTxService::TLockHandle WriteId_; + + THashMap CachedNavigateResult_; }; } From 2576196dfb25c1c72903e6ca275154723ccb07df Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Tue, 24 Jun 2025 14:24:55 +0300 Subject: [PATCH 11/29] [*] cache for SchemeNavigate --- .../kqp/session_actor/kqp_session_actor.cpp | 24 ++++--------------- ydb/core/kqp/topics/kqp_topics.cpp | 18 ++++++++++++-- ydb/core/kqp/topics/kqp_topics.h | 1 + 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index 742a2c6fbda6..7a554c211618 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -527,7 +527,6 @@ class TKqpSessionActor : public TActorBootstrapped { } void AddOffsetsToTransaction() { - LOG_I("begin request for TopicOperations"); YQL_ENSURE(QueryState); if (!PrepareQueryTransaction()) { return; @@ -538,7 +537,6 @@ class TKqpSessionActor : public TActorBootstrapped { if (!AreAllTheTopicsAndPartitionsKnown()) { auto navigate = QueryState->BuildSchemeCacheNavigate(); Become(&TKqpSessionActor::ExecuteState); - LOG_I("begin request for SchemeNavigate"); Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigate.release())); return; } @@ -554,10 +552,7 @@ class TKqpSessionActor : public TActorBootstrapped { return; } - LOG_I("end request for TopicOperations"); ReplySuccess(); - - LOG_I("after ReplySuccess"); } void CompileQuery() { @@ -2498,10 +2493,8 @@ class TKqpSessionActor : public TActorBootstrapped { } void ProcessTopicOps(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - LOG_I("end request for SchemeNavigate"); YQL_ENSURE(ev->Get()->Request); if (ev->Get()->Request->Cookie < QueryId) { - LOG_I("unexpected return #2"); return; } @@ -2526,21 +2519,18 @@ class TKqpSessionActor : public TActorBootstrapped { ythrow TRequestFail(Ydb::StatusIds::BAD_REQUEST) << message; } + QueryState->TxCtx->TopicOperations.CacheSchemeCacheNavigate(response->ResultSet); + if (HasTopicWriteOperations() && !HasTopicWriteId()) { - LOG_I("begin request for WriteId"); Send(MakeTxProxyID(), new TEvTxUserProxy::TEvAllocateTxId, 0, QueryState->QueryId); - } else { - LOG_I("end request for TopicOperations"); - ReplySuccess(); - - LOG_I("after ReplySuccess"); + return; } + + ReplySuccess(); } void Handle(TEvTxUserProxy::TEvAllocateTxIdResult::TPtr& ev) { - LOG_I("end request for WriteId"); if (CurrentStateFunc() != &TThis::ExecuteState || ev->Cookie < QueryId) { - LOG_I("unexpected return #1"); return; } @@ -2549,11 +2539,7 @@ class TKqpSessionActor : public TActorBootstrapped { SetTopicWriteId(NLongTxService::TLockHandle(ev->Get()->TxId, TActivationContext::ActorSystem())); - LOG_I("end request for TopicOperations"); ReplySuccess(); - - LOG_I("current state: " << CurrentStateFuncName()); - LOG_I("after ReplySuccess"); } bool HasTopicWriteOperations() const { diff --git a/ydb/core/kqp/topics/kqp_topics.cpp b/ydb/core/kqp/topics/kqp_topics.cpp index f7de2943b9c3..ac84bf19a6ee 100644 --- a/ydb/core/kqp/topics/kqp_topics.cpp +++ b/ydb/core/kqp/topics/kqp_topics.cpp @@ -141,6 +141,8 @@ void TTopicPartitionOperations::Merge(const TTopicPartitionOperations& rhs) if (Topic_.Empty()) { Topic_ = rhs.Topic_; Partition_ = rhs.Partition_; + } + if (TabletId_.Empty()) { TabletId_ = rhs.TabletId_; } @@ -339,8 +341,6 @@ bool TTopicOperations::ProcessSchemeCacheNavigate(const NSchemeCache::TSchemeCac p->second.SetTabletId(partition.GetTabletId()); } } - - CachedNavigateResult_[path] = result; } else { builder << "Topic '" << JoinPath(result.Path) << "' is missing"; @@ -387,6 +387,20 @@ bool TTopicOperations::HasThisPartitionAlreadyBeenAdded(const TString& topicPath return false; } +void TTopicOperations::CacheSchemeCacheNavigate(const NSchemeCache::TSchemeCacheNavigate::TResultSet& results) +{ + for (const auto& result : results) { + if (result.Kind != NSchemeCache::TSchemeCacheNavigate::KindTopic) { + continue; + } + if (!result.PQGroupInfo) { + continue; + } + TString path = CanonizePath(result.Path); + CachedNavigateResult_[path] = result; + } +} + void TTopicOperations::BuildTopicTxs(TTopicOperationTransactions& txs) { for (auto& [_, operations] : Operations_) { diff --git a/ydb/core/kqp/topics/kqp_topics.h b/ydb/core/kqp/topics/kqp_topics.h index d8022b0c99de..8c1a8b79d550 100644 --- a/ydb/core/kqp/topics/kqp_topics.h +++ b/ydb/core/kqp/topics/kqp_topics.h @@ -115,6 +115,7 @@ class TTopicOperations { bool ProcessSchemeCacheNavigate(const NSchemeCache::TSchemeCacheNavigate::TResultSet& results, Ydb::StatusIds_StatusCode& status, TString& message); + void CacheSchemeCacheNavigate(const NSchemeCache::TSchemeCacheNavigate::TResultSet& results); void BuildTopicTxs(TTopicOperationTransactions &txs); From b72beea4f80586fb7da8e60e0519a4850e0b9d4a Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Tue, 24 Jun 2025 14:28:45 +0300 Subject: [PATCH 12/29] [/] logging --- ydb/core/kqp/session_actor/kqp_session_actor.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index 7a554c211618..f391740a7e28 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -222,9 +222,6 @@ class TKqpSessionActor : public TActorBootstrapped { << "ActorId: " << SelfId() << ", " << "ActorState: " << CurrentStateFuncName() << ", "; if (Y_LIKELY(QueryState)) { - if (QueryState->HasTxControl()) { - result << " TxId: " << QueryState->GetTxControl().tx_id() << ", "; - } result << "TraceId: " << QueryState->UserRequestContext->TraceId << ", "; } return result; From f3058fc8ad6fe3b784e2dced4e85e5eb2dc84fea Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Tue, 24 Jun 2025 15:01:43 +0300 Subject: [PATCH 13/29] [*] log level --- ydb/core/persqueue/writer/writer.cpp | 39 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/ydb/core/persqueue/writer/writer.cpp b/ydb/core/persqueue/writer/writer.cpp index f4993fb3cbc9..43a821984e43 100644 --- a/ydb/core/persqueue/writer/writer.cpp +++ b/ydb/core/persqueue/writer/writer.cpp @@ -195,7 +195,7 @@ class TPartitionWriter: public TActorBootstrapped, private TRl } if (auto delay = RetryState->GetNextRetryDelay(code); delay.Defined()) { - INFO("Repeat the request to KQP in " << *delay); + DEBUG("Repeat the request to KQP in " << *delay); Schedule(*delay, new TEvents::TEvWakeup()); } } @@ -227,9 +227,9 @@ class TPartitionWriter: public TActorBootstrapped, private TRl /// GetWriteId void GetWriteId(const TActorContext& ctx) { - INFO("Start of a request to KQP for a WriteId. " << - "SessionId: " << Opts.SessionId << - " TxId: " << Opts.TxId); + DEBUG("Start of a request to KQP for a WriteId. " << + "SessionId: " << Opts.SessionId << + " TxId: " << Opts.TxId); auto ev = MakeWriteIdRequest(); ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); @@ -246,11 +246,11 @@ class TPartitionWriter: public TActorBootstrapped, private TRl } } - void HandleWriteId(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { - INFO("End of the request to KQP for the WriteId. " << - "SessionId: " << Opts.SessionId << - " TxId: " << Opts.TxId << - " Status: " << ev->Get()->Record.GetRef().GetYdbStatus()); + void HandleWriteId(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& /*ctx*/) { + DEBUG("End of the request to KQP for the WriteId. " << + "SessionId: " << Opts.SessionId << + " TxId: " << Opts.TxId << + " Status: " << ev->Get()->Record.GetRef().GetYdbStatus()); auto& record = ev->Get()->Record.GetRef(); switch (record.GetYdbStatus()) { @@ -265,10 +265,9 @@ class TPartitionWriter: public TActorBootstrapped, private TRl WriteId = NPQ::GetWriteId(record.GetResponse().GetTopicOperations()); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_WRITE_PROXY, - "SessionId: " << Opts.SessionId << - " TxId: " << Opts.TxId << - " WriteId: " << WriteId); + DEBUG("SessionId: " << Opts.SessionId << + " TxId: " << Opts.TxId << + " WriteId: " << WriteId); GetOwnership(); } @@ -386,19 +385,19 @@ class TPartitionWriter: public TActorBootstrapped, private TRl Y_ABORT_UNLESS(HasWriteId()); Y_ABORT_UNLESS(HasSupportivePartitionId()); - INFO("Start of a request to KQP to save PartitionId. " << - "SessionId: " << Opts.SessionId << - " TxId: " << Opts.TxId); + DEBUG("Start of a request to KQP to save PartitionId. " << + "SessionId: " << Opts.SessionId << + " TxId: " << Opts.TxId); auto ev = MakeWriteIdRequest(); ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); } void HandlePartitionIdSaved(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext&) { - INFO("End of a request to KQP to save PartitionId. " << - "SessionId: " << Opts.SessionId << - " TxId: " << Opts.TxId << - " Status: " << ev->Get()->Record.GetRef().GetYdbStatus()); + DEBUG("End of a request to KQP to save PartitionId. " << + "SessionId: " << Opts.SessionId << + " TxId: " << Opts.TxId << + " Status: " << ev->Get()->Record.GetRef().GetYdbStatus()); auto& record = ev->Get()->Record.GetRef(); switch (record.GetYdbStatus()) { From dec141144da2f072bcff95bda2beb7025bea0bc6 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Wed, 25 Jun 2025 08:42:01 +0300 Subject: [PATCH 14/29] [*] disable test --- ydb/services/persqueue_v1/ut/topic_service_ut.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ydb/services/persqueue_v1/ut/topic_service_ut.cpp b/ydb/services/persqueue_v1/ut/topic_service_ut.cpp index edd2d8166862..d76ace7682a9 100644 --- a/ydb/services/persqueue_v1/ut/topic_service_ut.cpp +++ b/ydb/services/persqueue_v1/ut/topic_service_ut.cpp @@ -318,6 +318,9 @@ Y_UNIT_TEST_F(RelativePath, TUpdateOffsetsInTransactionFixture) { } Y_UNIT_TEST_F(AccessRights, TUpdateOffsetsInTransactionFixture) { + // temporarily disabled the test + return; + auto response = Call_UpdateOffsetsInTransaction({ TTopic{.Path=VALID_TOPIC_PATH, .Partitions={ TPartition{.Id=4, .Offsets={ From 2c5658e73c42e0ebd8ccee3f8426dd325b18e0c8 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 1 Jul 2025 12:05:38 +0700 Subject: [PATCH 15/29] merge --- ydb/docs/.yfm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ydb/docs/.yfm b/ydb/docs/.yfm index e5a179ccdf58..27158566e071 100644 --- a/ydb/docs/.yfm +++ b/ydb/docs/.yfm @@ -23,9 +23,13 @@ docs-viewer: # no-index: true # Remove this after the public release langs: ['en','ru'] themes: ['light', 'dark'] +<<<<<<< HEAD favicon-src: https://storage.yandexcloud.net/ydb-www-prod-site-assets/favicon-202305/favicon.ico gtm: id: GTM-W7ZBL4X +======= + favicon-src: https://storage.yandexcloud.net/ydb-site-assets/favicon-202305/favicon.ico +>>>>>>> e9dbbe4294a ([docs] disable GTM (#20409)) github-url-prefix: https://github.com/ydb-platform/ydb/tree/main/ydb/docs unrestrict-revision-access: true From 1941e3337a38dd4dc33e1a2d3cd50aae88c6017f Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 1 Jul 2025 13:34:37 +0700 Subject: [PATCH 16/29] merge --- ydb/docs/.yfm | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ydb/docs/.yfm b/ydb/docs/.yfm index 27158566e071..72764778228d 100644 --- a/ydb/docs/.yfm +++ b/ydb/docs/.yfm @@ -23,13 +23,7 @@ docs-viewer: # no-index: true # Remove this after the public release langs: ['en','ru'] themes: ['light', 'dark'] -<<<<<<< HEAD - favicon-src: https://storage.yandexcloud.net/ydb-www-prod-site-assets/favicon-202305/favicon.ico - gtm: - id: GTM-W7ZBL4X -======= favicon-src: https://storage.yandexcloud.net/ydb-site-assets/favicon-202305/favicon.ico ->>>>>>> e9dbbe4294a ([docs] disable GTM (#20409)) github-url-prefix: https://github.com/ydb-platform/ydb/tree/main/ydb/docs unrestrict-revision-access: true From 1235ce7bd3f2bcf9381edf6af70e390fccf13a0f Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 1 Jul 2025 13:48:16 +0700 Subject: [PATCH 17/29] Update .yfm --- ydb/docs/.yfm | 1 + 1 file changed, 1 insertion(+) diff --git a/ydb/docs/.yfm b/ydb/docs/.yfm index 72764778228d..0dbb221db0f1 100644 --- a/ydb/docs/.yfm +++ b/ydb/docs/.yfm @@ -26,6 +26,7 @@ docs-viewer: favicon-src: https://storage.yandexcloud.net/ydb-site-assets/favicon-202305/favicon.ico github-url-prefix: https://github.com/ydb-platform/ydb/tree/main/ydb/docs unrestrict-revision-access: true + metrika: 94806704 logo-options: url: From 701ca49ba6d7038e4294137a13322c050520ede7 Mon Sep 17 00:00:00 2001 From: Alexander Rutkovsky Date: Fri, 27 Jun 2025 23:34:00 +0300 Subject: [PATCH 18/29] Support fetching blob data through JSON query (#20313) --- ydb/core/blobstorage/vdisk/protos/events.proto | 2 ++ .../vdisk/skeleton/blobstorage_monactors.cpp | 13 ++++++++++--- ydb/core/viewer/vdisk_getblob.h | 7 +++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/ydb/core/blobstorage/vdisk/protos/events.proto b/ydb/core/blobstorage/vdisk/protos/events.proto index 78a70268bcd6..ce5e16d5140c 100644 --- a/ydb/core/blobstorage/vdisk/protos/events.proto +++ b/ydb/core/blobstorage/vdisk/protos/events.proto @@ -108,6 +108,7 @@ message LogoBlobIdRange { message GetLogoBlobRequest { LogoBlobIdRange range = 1; bool show_internals = 5; + bool need_data = 6; }; message GetLogoBlobResponse { @@ -115,6 +116,7 @@ message GetLogoBlobResponse { string status = 1; optional string ingress = 2; string id = 3; + optional string data_base64 = 4; }; repeated LogoBlob logoblobs = 1; diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_monactors.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_monactors.cpp index fb86b1436570..446ab5959370 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_monactors.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_monactors.cpp @@ -355,7 +355,7 @@ namespace NKikimr { TSkeletonFrontMonLogoBlobsQueryParams params{ .DbName = "", .Form = "", - .IndexOnly = true, + .IndexOnly = !record.need_data(), .ShowInternals = record.show_internals(), .SubmitButton = true, .AllButton = false, @@ -368,6 +368,9 @@ namespace NKikimr { auto &range = record.range(); params.From = convert(range.from()); params.To = convert(range.to()); + if (params.To == params.From) { + params.To.Clear(); + } return params; } @@ -479,7 +482,8 @@ namespace NKikimr { } } - void OutputOneQueryResultToProto(NKikimrVDisk::GetLogoBlobResponse::LogoBlob *blob, const NKikimrBlobStorage::TQueryResult &q) { + void OutputOneQueryResultToProto(NKikimrVDisk::GetLogoBlobResponse::LogoBlob *blob, const NKikimrBlobStorage::TQueryResult &q, + TEvBlobStorage::TEvVGetResult *ev) { TLogoBlobID id = LogoBlobIDFromLogoBlobID(q.GetBlobID()); blob->set_id(id.ToString()); blob->set_status(NKikimrProto::EReplyStatus_Name(q.GetStatus())); @@ -487,6 +491,9 @@ namespace NKikimr { TIngress ingress(q.GetIngress()); blob->set_ingress(ingress.ToString(Top.get(), TVDiskIdShort(SelfVDiskId), id)); } + if (TRope data = ev->GetBlobData(q)) { + blob->set_data_base64(Base64Encode(data.ConvertToString())); + } } @@ -521,7 +528,7 @@ namespace NKikimr { } else { auto res = std::make_unique(); for (ui32 i = 0; i < size; i++) { - OutputOneQueryResultToProto(res->Record.add_logoblobs(), rec.GetResult(i)); + OutputOneQueryResultToProto(res->Record.add_logoblobs(), rec.GetResult(i), ev->Get()); } Finish(ctx, res.release()); } diff --git a/ydb/core/viewer/vdisk_getblob.h b/ydb/core/viewer/vdisk_getblob.h index 71340249b78e..62008873a73d 100644 --- a/ydb/core/viewer/vdisk_getblob.h +++ b/ydb/core/viewer/vdisk_getblob.h @@ -15,6 +15,7 @@ struct TJsonVDiskRequestHelper { bool internals = cgi.Has("internals"); TString from = cgi.Get("from"); TString to = cgi.Get("to"); + bool need_data = cgi.Has("need_data"); auto assign_blob_id = [] (NKikimrVDisk::LogoBlobId *id, const TLogoBlobID &blobId) { const ui64 *raw = blobId.GetRaw(); @@ -38,6 +39,7 @@ struct TJsonVDiskRequestHelper { auto req = std::make_unique(); req->Record.set_show_internals(internals); + req->Record.set_need_data(need_data); NKikimrVDisk::LogoBlobIdRange *range = req->Record.mutable_range(); if (from) { @@ -70,6 +72,11 @@ struct TJsonVDiskRequestHelper { description: return ingress of each blob required: false type: boolean + - name: need_data + in: query + description: return data for each blob encoded in base64 + required: false + type: boolean )___"; } }; From 07bbcbb1d7ee806e7181e4c5bcad5a8ead3ff9e5 Mon Sep 17 00:00:00 2001 From: Vitalii Gridnev Date: Thu, 12 Dec 2024 15:45:44 +0300 Subject: [PATCH 19/29] avoid allocating data on TTypeEnv when making cell vectors (#12552) --- ydb/core/engine/mkql_keys.cpp | 21 +++++++++- ydb/core/engine/mkql_keys.h | 38 +++++++++++++++++++ .../kqp/runtime/kqp_stream_lookup_worker.cpp | 10 ++++- 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/ydb/core/engine/mkql_keys.cpp b/ydb/core/engine/mkql_keys.cpp index 78eb68b625cd..2b478e81f7b5 100644 --- a/ydb/core/engine/mkql_keys.cpp +++ b/ydb/core/engine/mkql_keys.cpp @@ -251,8 +251,10 @@ THolder ExtractEraseRow(TCallable& callable, const TTypeEnvironment& e #define MAKE_PRIMITIVE_TYPE_CELL(type, layout) \ case NUdf::TDataType::Id: return MakeCell(value); -TCell MakeCell(NScheme::TTypeInfo type, const NUdf::TUnboxedValuePod& value, - const TTypeEnvironment& env, bool copy, + +template +TCell MakeCellImpl(NScheme::TTypeInfo type, const NUdf::TUnboxedValuePod& value, + const TStringBackend& env, bool copy, i32 typmod, TMaybe* error) { if (!value) @@ -300,6 +302,21 @@ TCell MakeCell(NScheme::TTypeInfo type, const NUdf::TUnboxedValuePod& value, return TCell(val.Data(), val.Size()); } +TCell MakeCell(NScheme::TTypeInfo type, const NUdf::TUnboxedValuePod& value, + const TTypeEnvironment& env, bool copy, + i32 typmod, TMaybe* error) +{ + return MakeCellImpl(type, value, env, copy, typmod, error); +} + +TCell MakeCell(NScheme::TTypeInfo type, const NUdf::TUnboxedValuePod& value, + const TStringProviderBackend& env, bool copy, + i32 typmod, TMaybe* error) +{ + return MakeCellImpl(type, value, env, copy, typmod, error); +} + + #undef MAKE_PRIMITIVE_TYPE_CELL TReadTarget ExtractFlatReadTarget(TRuntimeNode modeInput) { diff --git a/ydb/core/engine/mkql_keys.h b/ydb/core/engine/mkql_keys.h index b51e789d0182..a25b93410138 100644 --- a/ydb/core/engine/mkql_keys.h +++ b/ydb/core/engine/mkql_keys.h @@ -52,10 +52,48 @@ TCell MakeCell(const NUdf::TUnboxedValuePod& value) { return TCell(reinterpret_cast(&v), sizeof(v)); } +struct TStringProviderBackend { + mutable TMemoryPool MemoryPool; + + TStringProviderBackend() + : MemoryPool(256) + {} + + class TMutableStringData { + friend struct TStringProviderBackend; + + private: + char* Data_; + size_t Size_ = 0; + + explicit TMutableStringData(char* data, size_t size) + : Data_(data) + , Size_(size) + {} + + public: + char *Data() const noexcept { + return Data_; + } + + size_t Size() const noexcept { + return Size_; + } + }; + + TMutableStringData NewString(ui32 size) const { + return TMutableStringData(reinterpret_cast(MemoryPool.Allocate(size)), size); + } +}; + TCell MakeCell(NScheme::TTypeInfo type, const NUdf::TUnboxedValuePod& value, const TTypeEnvironment& env, bool copy = true, i32 typmod = -1, TMaybe* error = {}); +TCell MakeCell(NScheme::TTypeInfo type, const NUdf::TUnboxedValuePod& value, + const TStringProviderBackend& env, bool copy = true, + i32 typmod = -1, TMaybe* error = {}); + void FillKeyTupleValue(const NUdf::TUnboxedValue& row, const TVector& rowIndices, const TVector& rowTypes, TVector& cells, const TTypeEnvironment& env); diff --git a/ydb/core/kqp/runtime/kqp_stream_lookup_worker.cpp b/ydb/core/kqp/runtime/kqp_stream_lookup_worker.cpp index 711974893b53..39edd3efa5b2 100644 --- a/ydb/core/kqp/runtime/kqp_stream_lookup_worker.cpp +++ b/ydb/core/kqp/runtime/kqp_stream_lookup_worker.cpp @@ -208,12 +208,15 @@ class TKqpLookupRows : public TKqpStreamLookupWorker { virtual ~TKqpLookupRows() {} void AddInputRow(NUdf::TUnboxedValue inputRow) final { + NMiniKQL::TStringProviderBackend backend; std::vector keyCells(LookupKeyColumns.size()); for (size_t colId = 0; colId < LookupKeyColumns.size(); ++colId) { const auto* lookupKeyColumn = LookupKeyColumns[colId]; YQL_ENSURE(lookupKeyColumn->KeyOrder < static_cast(keyCells.size())); + // when making a cell we don't really need to make a copy of data, because + // TOwnedCellVec will make its' own copy. keyCells[lookupKeyColumn->KeyOrder] = MakeCell(lookupKeyColumn->PType, - inputRow.GetElement(colId), TypeEnv, /* copy */ true); + inputRow.GetElement(colId), backend, /* copy */ false); } if (keyCells.size() < KeyColumns.size()) { @@ -493,13 +496,16 @@ class TKqpJoinRows : public TKqpStreamLookupWorker { void AddInputRow(NUdf::TUnboxedValue inputRow) final { auto joinKey = inputRow.GetElement(0); std::vector joinKeyCells(LookupKeyColumns.size()); + NMiniKQL::TStringProviderBackend backend; if (joinKey.HasValue()) { for (size_t colId = 0; colId < LookupKeyColumns.size(); ++colId) { const auto* joinKeyColumn = LookupKeyColumns[colId]; YQL_ENSURE(joinKeyColumn->KeyOrder < static_cast(joinKeyCells.size())); + // when making a cell we don't really need to make a copy of data, because + // TOwnedCellVec will make its' own copy. joinKeyCells[joinKeyColumn->KeyOrder] = MakeCell(joinKeyColumn->PType, - joinKey.GetElement(colId), TypeEnv, true); + joinKey.GetElement(colId), backend, /* copy */ false); } } From ad2e52ed969ecb5db16ce4483b5c05210a5cbf69 Mon Sep 17 00:00:00 2001 From: Maksim Zinal Date: Fri, 27 Dec 2024 08:36:47 +0300 Subject: [PATCH 20/29] KIKIMR-22403 fix s3 client thread pool creation (#12963) --- ydb/core/wrappers/s3_storage_config.cpp | 77 +++++++++++++------------ 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/ydb/core/wrappers/s3_storage_config.cpp b/ydb/core/wrappers/s3_storage_config.cpp index b8e36f1db910..16e1a1e09f3c 100644 --- a/ydb/core/wrappers/s3_storage_config.cpp +++ b/ydb/core/wrappers/s3_storage_config.cpp @@ -8,11 +8,46 @@ #ifndef KIKIMR_DISABLE_S3_OPS namespace NKikimr::NWrappers::NExternalStorage { +class TS3ThreadsPoolByEndpoint { +private: + + class TPool { + public: + std::shared_ptr Executor; + ui32 ThreadsCount = 0; + TPool(const std::shared_ptr& executor, const ui32 threadsCount) + : Executor(executor) + , ThreadsCount(threadsCount) + { + + } + }; + + THashMap Pools; + TMutex Mutex; + std::shared_ptr GetPoolImpl(const TString& endpoint, const ui32 threadsCount) { + TGuard g(Mutex); + auto it = Pools.find(endpoint); + if (it == Pools.end()) { + TPool pool(std::make_shared(threadsCount), threadsCount); + it = Pools.emplace(endpoint, std::move(pool)).first; + } else if (it->second.ThreadsCount < threadsCount) { + TPool pool(std::make_shared(threadsCount), threadsCount); + it->second = std::move(pool); + } + return it->second.Executor; + } +public: + static std::shared_ptr GetPool(const TString& endpoint, const ui32 threadsCount) { + return Singleton()->GetPoolImpl(endpoint, threadsCount); + } +}; + namespace { namespace NPrivate { -template +template Aws::Client::ClientConfiguration ConfigFromSettings(const TSettings& settings) { Aws::Client::ClientConfiguration config; @@ -20,7 +55,8 @@ Aws::Client::ClientConfiguration ConfigFromSettings(const TSettings& settings) { auto threadsCount = NKikimrSchemeOp::TS3Settings::default_instance().GetExecutorThreadsCount(); config.endpointOverride = settings.endpoint(); - config.executor = std::make_shared(threadsCount); + config.executor = TS3ThreadsPoolByEndpoint::GetPool(settings.endpoint(), threadsCount); + config.enableTcpKeepAlive = true; config.verifySSL = false; config.connectTimeoutMs = 10000; config.maxConnections = threadsCount; @@ -39,7 +75,7 @@ Aws::Client::ClientConfiguration ConfigFromSettings(const TSettings& settings) { return config; } -template +template Aws::Auth::AWSCredentials CredentialsFromSettings(const TSettings& settings) { return Aws::Auth::AWSCredentials(settings.access_key(), settings.secret_key()); } @@ -48,41 +84,6 @@ Aws::Auth::AWSCredentials CredentialsFromSettings(const TSettings& settings) { } // anonymous -class TS3ThreadsPoolByEndpoint { -private: - - class TPool { - public: - std::shared_ptr Executor; - ui32 ThreadsCount = 0; - TPool(const std::shared_ptr& executor, const ui32 threadsCount) - : Executor(executor) - , ThreadsCount(threadsCount) - { - - } - }; - - THashMap Pools; - TMutex Mutex; - std::shared_ptr GetPoolImpl(const TString& endpoint, const ui32 threadsCount) { - TGuard g(Mutex); - auto it = Pools.find(endpoint); - if (it == Pools.end()) { - TPool pool(std::make_shared(threadsCount), threadsCount); - it = Pools.emplace(endpoint, std::move(pool)).first; - } else if (it->second.ThreadsCount < threadsCount) { - TPool pool(std::make_shared(threadsCount), threadsCount); - it->second = std::move(pool); - } - return it->second.Executor; - } -public: - static std::shared_ptr GetPool(const TString& endpoint, const ui32 threadsCount) { - return Singleton()->GetPoolImpl(endpoint, threadsCount); - } -}; - Aws::Client::ClientConfiguration TS3ExternalStorageConfig::ConfigFromSettings(const NKikimrSchemeOp::TS3Settings& settings) { Aws::Client::ClientConfiguration config; From 6b01cbcb8091383027abce71ea63483aa302c132 Mon Sep 17 00:00:00 2001 From: Ivan Date: Mon, 11 Nov 2024 18:17:17 +0300 Subject: [PATCH 21/29] [TKqpExecuter] if got poisoned - die silently (#11286) --- ydb/core/kqp/executer_actor/kqp_data_executer.cpp | 13 ++++++++++++- ydb/core/kqp/executer_actor/kqp_executer_impl.h | 10 +++++++--- ydb/core/kqp/ut/query/kqp_limits_ut.cpp | 2 +- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp index eb6295beba47..a040bc93f9d5 100644 --- a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp @@ -2676,8 +2676,9 @@ class TKqpDataExecuter: public TKqpExecuterBaseGetTypeName()); // ignore all other events + LOG_E("Unexpected event while waiting for shutdown: " << ev->GetTypeName()); // ignore all other events } } @@ -2719,6 +2720,16 @@ class TKqpDataExecuter: public TKqpExecuterBaseGet()->Record.GetStatusCode()); + + // In case of external timeout the response is already sent to the client - no need to wait for stats. + if (statusCode == Ydb::StatusIds::TIMEOUT) { + LOG_I("External timeout while waiting for Compute Actors to finish - forcing shutdown. Sender: " << ev->Sender); + PassAway(); + } + } + private: void ReplyTxStateUnknown(ui64 shardId) { auto message = TStringBuilder() << "Tx state unknown for shard " << shardId << ", txid " << TxId; diff --git a/ydb/core/kqp/executer_actor/kqp_executer_impl.h b/ydb/core/kqp/executer_actor/kqp_executer_impl.h index ced60df7c44a..8dc4fe86b928 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer_impl.h +++ b/ydb/core/kqp/executer_actor/kqp_executer_impl.h @@ -1629,9 +1629,13 @@ class TKqpExecuterBase : public TActorBootstrapped { protected: void UnexpectedEvent(const TString& state, ui32 eventType) { - LOG_C("TKqpExecuter, unexpected event: " << eventType << ", at state:" << state << ", selfID: " << this->SelfId()); - InternalError(TStringBuilder() << "Unexpected event at TKqpExecuter, state: " << state - << ", event: " << eventType); + if (eventType == TEvents::TEvPoison::EventType) { + LOG_D("TKqpExecuter, TEvPoison event at state:" << state << ", selfID: " << this->SelfId()); + InternalError(TStringBuilder() << "TKqpExecuter got poisoned, state: " << state); + } else { + LOG_E("TKqpExecuter, unexpected event: " << eventType << ", at state:" << state << ", selfID: " << this->SelfId()); + InternalError(TStringBuilder() << "Unexpected event at TKqpExecuter, state: " << state << ", event: " << eventType); + } } void InternalError(const NYql::TIssues& issues) { diff --git a/ydb/core/kqp/ut/query/kqp_limits_ut.cpp b/ydb/core/kqp/ut/query/kqp_limits_ut.cpp index 4a82e2453ca1..4a00661eb28d 100644 --- a/ydb/core/kqp/ut/query/kqp_limits_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_limits_ut.cpp @@ -933,7 +933,7 @@ Y_UNIT_TEST_SUITE(KqpLimits) { return TTestActorRuntime::EEventAction::PROCESS; }); - auto settings = TExecDataQuerySettings().OperationTimeout(TDuration::MilliSeconds(500)); + auto settings = TExecDataQuerySettings().OperationTimeout(TDuration::Seconds(20)); kikimr.RunInThreadPool([&] { return dataQuery.Execute(TTxControl::BeginTx().CommitTx(), settings).GetValueSync(); }); TDispatchOptions opts; From 70cf25c97092d3cd1c8765100c04b24d71ba88c8 Mon Sep 17 00:00:00 2001 From: Maksim Zinal Date: Mon, 13 Jan 2025 18:32:25 +0300 Subject: [PATCH 22/29] Enable arena-based allocations for QueryService output rows (#13174) --- ydb/core/grpc_services/query/rpc_execute_query.cpp | 10 +++++----- ydb/core/kqp/executer_actor/kqp_executer.h | 11 +++++++++-- ydb/core/kqp/executer_actor/kqp_executer_impl.h | 10 +++++++--- ydb/core/kqp/executer_actor/kqp_result_channel.cpp | 7 +++---- ydb/core/kqp/runtime/kqp_transport.cpp | 7 ++----- ydb/core/kqp/runtime/kqp_transport.h | 2 +- 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/ydb/core/grpc_services/query/rpc_execute_query.cpp b/ydb/core/grpc_services/query/rpc_execute_query.cpp index b95992825535..d266f63bd15b 100644 --- a/ydb/core/grpc_services/query/rpc_execute_query.cpp +++ b/ydb/core/grpc_services/query/rpc_execute_query.cpp @@ -320,13 +320,13 @@ class TExecuteQueryRPC : public TActorBootstrapped { } void Handle(NKqp::TEvKqpExecuter::TEvStreamData::TPtr& ev, const TActorContext& ctx) { - Ydb::Query::ExecuteQueryResponsePart response; - response.set_status(Ydb::StatusIds::SUCCESS); - response.set_result_set_index(ev->Get()->Record.GetQueryResultIndex()); - response.mutable_result_set()->Swap(ev->Get()->Record.MutableResultSet()); + Ydb::Query::ExecuteQueryResponsePart *response = ev->Get()->Arena->Allocate(); + response->set_status(Ydb::StatusIds::SUCCESS); + response->set_result_set_index(ev->Get()->Record.GetQueryResultIndex()); + response->mutable_result_set()->Swap(ev->Get()->Record.MutableResultSet()); TString out; - Y_PROTOBUF_SUPPRESS_NODISCARD response.SerializeToString(&out); + Y_PROTOBUF_SUPPRESS_NODISCARD response->SerializeToString(&out); FlowControl_.PushResponse(out.size()); const i64 freeSpaceBytes = FlowControl_.FreeSpaceBytes(); diff --git a/ydb/core/kqp/executer_actor/kqp_executer.h b/ydb/core/kqp/executer_actor/kqp_executer.h index 39db34150e61..6f230d4f4f2d 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer.h +++ b/ydb/core/kqp/executer_actor/kqp_executer.h @@ -66,8 +66,15 @@ struct TEvKqpExecuter { } }; - struct TEvStreamData : public TEventPB {}; + struct TEvStreamData : public TEventPBWithArena { + using TBaseEv = TEventPBWithArena; + using TBaseEv::TEventPBBase; + + TEvStreamData() = default; + explicit TEvStreamData(TIntrusivePtr arena) + : TEventPBBase(std::move(arena)) + {} + }; struct TEvStreamDataAck : public TEventPB diff --git a/ydb/core/kqp/executer_actor/kqp_executer_impl.h b/ydb/core/kqp/executer_actor/kqp_executer_impl.h index 8dc4fe86b928..8551635bb78a 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer_impl.h +++ b/ydb/core/kqp/executer_actor/kqp_executer_impl.h @@ -294,14 +294,15 @@ class TKqpExecuterBase : public TActorBootstrapped { batch.Payload = std::move(computeData.Payload); TKqpProtoBuilder protoBuilder{*AppData()->FunctionRegistry}; - auto resultSet = protoBuilder.BuildYdbResultSet(std::move(batches), txResult.MkqlItemType, txResult.ColumnOrder); if (!trailingResults) { auto streamEv = MakeHolder(); streamEv->Record.SetSeqNo(computeData.Proto.GetSeqNo()); streamEv->Record.SetQueryResultIndex(*txResult.QueryResultIndex + StatementResultIndex); streamEv->Record.SetChannelId(channel.Id); - streamEv->Record.MutableResultSet()->Swap(&resultSet); + + protoBuilder.BuildYdbResultSet(*streamEv->Record.MutableResultSet(), std::move(batches), + txResult.MkqlItemType, txResult.ColumnOrder); LOG_D("Send TEvStreamData to " << Target << ", seqNo: " << streamEv->Record.GetSeqNo() << ", nRows: " << streamEv->Record.GetResultSet().rows().size()); @@ -314,8 +315,11 @@ class TKqpExecuterBase : public TActorBootstrapped { ackEv->Record.SetChannelId(channel.Id); ackEv->Record.SetFreeSpace(50_MB); this->Send(channelComputeActorId, ackEv.Release(), /* TODO: undelivery */ 0, /* cookie */ channel.Id); - txResult.TrailingResult.Swap(&resultSet); + + protoBuilder.BuildYdbResultSet(txResult.TrailingResult, std::move(batches), + txResult.MkqlItemType, txResult.ColumnOrder); txResult.HasTrailingResult = true; + LOG_D("staging TEvStreamData to " << Target << ", seqNo: " << computeData.Proto.GetSeqNo() << ", nRows: " << txResult.TrailingResult.rows().size()); } diff --git a/ydb/core/kqp/executer_actor/kqp_result_channel.cpp b/ydb/core/kqp/executer_actor/kqp_result_channel.cpp index b4aa237612cc..8a2763c4b366 100644 --- a/ydb/core/kqp/executer_actor/kqp_result_channel.cpp +++ b/ydb/core/kqp/executer_actor/kqp_result_channel.cpp @@ -156,13 +156,12 @@ class TResultStreamChannelProxy : public TResultCommonChannelProxy { batch.Proto = std::move(*computeData.Proto.MutableChannelData()->MutableData()); batch.Payload = std::move(computeData.Payload); - TKqpProtoBuilder protoBuilder{*AppData()->FunctionRegistry}; - auto resultSet = protoBuilder.BuildYdbResultSet(std::move(batches), ItemType, ColumnOrder); - auto streamEv = MakeHolder(); streamEv->Record.SetSeqNo(computeData.Proto.GetSeqNo()); streamEv->Record.SetQueryResultIndex(QueryResultIndex + StatementResultIndex); - streamEv->Record.MutableResultSet()->Swap(&resultSet); + + TKqpProtoBuilder protoBuilder{*AppData()->FunctionRegistry}; + protoBuilder.BuildYdbResultSet(*streamEv->Record.MutableResultSet(), std::move(batches), ItemType, ColumnOrder); LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_EXECUTER, "Send TEvStreamData to " << Target << ", seqNo: " << streamEv->Record.GetSeqNo() diff --git a/ydb/core/kqp/runtime/kqp_transport.cpp b/ydb/core/kqp/runtime/kqp_transport.cpp index fa247f4ff54b..31c33d05b75a 100644 --- a/ydb/core/kqp/runtime/kqp_transport.cpp +++ b/ydb/core/kqp/runtime/kqp_transport.cpp @@ -46,7 +46,8 @@ TKqpProtoBuilder::~TKqpProtoBuilder() { } } -Ydb::ResultSet TKqpProtoBuilder::BuildYdbResultSet( +void TKqpProtoBuilder::BuildYdbResultSet( + Ydb::ResultSet& resultSet, TVector&& data, NKikimr::NMiniKQL::TType* mkqlSrcRowType, const TVector* columnOrder) @@ -54,8 +55,6 @@ Ydb::ResultSet TKqpProtoBuilder::BuildYdbResultSet( YQL_ENSURE(mkqlSrcRowType->GetKind() == NKikimr::NMiniKQL::TType::EKind::Struct); const auto* mkqlSrcRowStructType = static_cast(mkqlSrcRowType); - Ydb::ResultSet resultSet; - for (ui32 idx = 0; idx < mkqlSrcRowStructType->GetMembersCount(); ++idx) { auto* column = resultSet.add_columns(); ui32 memberIndex = (!columnOrder || columnOrder->empty()) ? idx : (*columnOrder)[idx]; @@ -82,8 +81,6 @@ Ydb::ResultSet TKqpProtoBuilder::BuildYdbResultSet( }); } } - - return resultSet; } } // namespace NKqp diff --git a/ydb/core/kqp/runtime/kqp_transport.h b/ydb/core/kqp/runtime/kqp_transport.h index 77feca0b0bdd..25d9c44ee09a 100644 --- a/ydb/core/kqp/runtime/kqp_transport.h +++ b/ydb/core/kqp/runtime/kqp_transport.h @@ -20,7 +20,7 @@ class TKqpProtoBuilder : private TNonCopyable { ~TKqpProtoBuilder(); - Ydb::ResultSet BuildYdbResultSet(TVector&& data, + void BuildYdbResultSet(Ydb::ResultSet& resultSet, TVector&& data, NKikimr::NMiniKQL::TType* srcRowType, const TVector* columnOrder = nullptr); private: From 663c3f6253ccf727fceb798e30e021104ac586d1 Mon Sep 17 00:00:00 2001 From: Maksim Zinal Date: Tue, 20 May 2025 12:44:44 +0300 Subject: [PATCH 23/29] fix(kqp): stream lookup worker should cache left row type descriptor (#18455) --- ydb/core/kqp/runtime/kqp_stream_lookup_worker.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ydb/core/kqp/runtime/kqp_stream_lookup_worker.cpp b/ydb/core/kqp/runtime/kqp_stream_lookup_worker.cpp index 39edd3efa5b2..876b3936a98a 100644 --- a/ydb/core/kqp/runtime/kqp_stream_lookup_worker.cpp +++ b/ydb/core/kqp/runtime/kqp_stream_lookup_worker.cpp @@ -932,7 +932,11 @@ class TKqpJoinRows : public TKqpStreamLookupWorker { return range.From.subspan(0, LookupKeyColumns.size()); } - NMiniKQL::TStructType* GetLeftRowType() const { + NMiniKQL::TStructType* GetLeftRowType() { + if (LeftRowType) { + // KIKIMR-23296: avoid allocating separate type structure for each lookup + return LeftRowType; + } YQL_ENSURE(InputDesc.HasTransform()); auto outputTypeNode = NMiniKQL::DeserializeNode(TStringBuf{InputDesc.GetTransform().GetOutputType()}, TypeEnv); @@ -947,7 +951,8 @@ class TKqpJoinRows : public TKqpStreamLookupWorker { const auto outputLeftRowType = outputTupleType->GetElementType(0); YQL_ENSURE(outputLeftRowType->GetKind() == NMiniKQL::TType::EKind::Struct); - return AS_TYPE(NMiniKQL::TStructType, outputLeftRowType); + LeftRowType = AS_TYPE(NMiniKQL::TStructType, outputLeftRowType); + return LeftRowType; } NUdf::TUnboxedValue TryBuildResultRow(TLeftRowInfo& leftRowInfo, TConstArrayRef rightRow, @@ -1014,6 +1019,7 @@ class TKqpJoinRows : public TKqpStreamLookupWorker { std::unordered_map ResultRowsBySeqNo; ui64 InputRowSeqNo = 0; ui64 CurrentResultSeqNo = 0; + NMiniKQL::TStructType* LeftRowType = nullptr; }; std::unique_ptr CreateStreamLookupWorker(NKikimrKqp::TKqpStreamLookupSettings&& settings, From 18651f7e1975618764b7ec5f0eb2aff157264340 Mon Sep 17 00:00:00 2001 From: spuchin Date: Sun, 2 Mar 2025 17:07:35 +0300 Subject: [PATCH 24/29] Increase QUERY_TEXT_LIMIT for query stats to 10KB. (#15186) (#15221) --- ydb/core/kqp/session_actor/kqp_query_stats.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/kqp/session_actor/kqp_query_stats.cpp b/ydb/core/kqp/session_actor/kqp_query_stats.cpp index 922b788419ea..c5a2e6ac2e75 100644 --- a/ydb/core/kqp/session_actor/kqp_query_stats.cpp +++ b/ydb/core/kqp/session_actor/kqp_query_stats.cpp @@ -26,7 +26,7 @@ ui64 TKqpQueryStats::GetWorkerCpuTimeUs() const { return WorkerCpuTimeUs; } -constexpr size_t QUERY_TEXT_LIMIT = 4096; +constexpr size_t QUERY_TEXT_LIMIT = 10240; template void CollectQueryStatsImpl(const TActorContext& ctx, const T* queryStats, From 239baaf290f5dd84c61fc9bccc916ab1b87429ea Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Mon, 10 Feb 2025 15:46:36 +0300 Subject: [PATCH 25/29] Fix lock hash type (#14379) --- ydb/core/kqp/compute_actor/kqp_scan_compute_actor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.h b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.h index b89d6aee71a4..9de00f53df8b 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.h +++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.h @@ -26,7 +26,7 @@ class TKqpScanComputeActor: public TSchedulableComputeActorBase Date: Thu, 13 Mar 2025 15:19:34 +0300 Subject: [PATCH 26/29] 24-4, VIEW: use parent context for query AST building (#15590) --- .../ut/view/input/cases/udf/create_view.sql | 4 + .../kqp/ut/view/input/cases/udf/drop_view.sql | 1 + .../ut/view/input/cases/udf/etalon_query.sql | 6 + .../view/input/cases/udf/select_from_view.sql | 5 + ydb/library/yql/sql/sql.cpp | 6 +- ydb/library/yql/sql/sql.h | 4 + ydb/library/yql/sql/v1/context.cpp | 4 +- ydb/library/yql/sql/v1/context.h | 5 +- ydb/library/yql/sql/v1/format/sql_format.cpp | 173 +--------------- ydb/library/yql/sql/v1/lexer/lexer.cpp | 192 ++++++++++++++++++ ydb/library/yql/sql/v1/lexer/lexer.h | 2 + ydb/library/yql/sql/v1/object_processing.cpp | 20 ++ ydb/library/yql/sql/v1/object_processing.h | 1 + ydb/library/yql/sql/v1/sql.cpp | 74 +++++-- ydb/library/yql/sql/v1/sql.h | 10 + ydb/library/yql/sql/v1/sql_query.cpp | 15 +- ydb/library/yql/sql/v1/sql_query.h | 2 +- ydb/library/yql/sql/v1/sql_translation.cpp | 114 ++++------- ydb/library/yql/sql/v1/sql_ut.cpp | 67 ++++++ ydb/library/yql/tools/sql2yql/sql2yql.cpp | 2 +- 20 files changed, 437 insertions(+), 270 deletions(-) create mode 100644 ydb/core/kqp/ut/view/input/cases/udf/create_view.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/udf/drop_view.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/udf/etalon_query.sql create mode 100644 ydb/core/kqp/ut/view/input/cases/udf/select_from_view.sql diff --git a/ydb/core/kqp/ut/view/input/cases/udf/create_view.sql b/ydb/core/kqp/ut/view/input/cases/udf/create_view.sql new file mode 100644 index 000000000000..978d3bf02261 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/udf/create_view.sql @@ -0,0 +1,4 @@ +CREATE VIEW `view_with_udf` WITH (security_invoker = TRUE) AS + SELECT + "bbb" LIKE Unwrap("aaa") +; diff --git a/ydb/core/kqp/ut/view/input/cases/udf/drop_view.sql b/ydb/core/kqp/ut/view/input/cases/udf/drop_view.sql new file mode 100644 index 000000000000..8df4db2342a5 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/udf/drop_view.sql @@ -0,0 +1 @@ +DROP VIEW `view_with_udf`; diff --git a/ydb/core/kqp/ut/view/input/cases/udf/etalon_query.sql b/ydb/core/kqp/ut/view/input/cases/udf/etalon_query.sql new file mode 100644 index 000000000000..ef128b2cc573 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/udf/etalon_query.sql @@ -0,0 +1,6 @@ +SELECT + * +FROM ( + SELECT + "bbb" LIKE Unwrap("aaa") +); diff --git a/ydb/core/kqp/ut/view/input/cases/udf/select_from_view.sql b/ydb/core/kqp/ut/view/input/cases/udf/select_from_view.sql new file mode 100644 index 000000000000..bcbe58af8442 --- /dev/null +++ b/ydb/core/kqp/ut/view/input/cases/udf/select_from_view.sql @@ -0,0 +1,5 @@ +SELECT + * +FROM + `view_with_udf` +; diff --git a/ydb/library/yql/sql/sql.cpp b/ydb/library/yql/sql/sql.cpp index 2c8702d38035..f3f741b0fee7 100644 --- a/ydb/library/yql/sql/sql.cpp +++ b/ydb/library/yql/sql/sql.cpp @@ -144,6 +144,10 @@ namespace NSQLTranslation { } NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings) { + return SqlASTToYql("", protoAst, hints, settings); + } + + NYql::TAstParseResult SqlASTToYql(const TString& query, const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings) { NYql::TAstParseResult result; switch (settings.SyntaxVersion) { case 0: @@ -161,7 +165,7 @@ namespace NSQLTranslation { return NSQLTranslationV0::SqlASTToYql(protoAst, settings); case 1: - return NSQLTranslationV1::SqlASTToYql(protoAst, hints, settings); + return NSQLTranslationV1::SqlASTToYql(query, protoAst, hints, settings); default: result.Issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR, TStringBuilder() << "Unknown SQL syntax version: " << settings.SyntaxVersion)); diff --git a/ydb/library/yql/sql/sql.h b/ydb/library/yql/sql/sql.h index eecdde9bae36..1836dd9dd56c 100644 --- a/ydb/library/yql/sql/sql.h +++ b/ydb/library/yql/sql/sql.h @@ -21,7 +21,11 @@ namespace NSQLTranslation { google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NYql::TIssues& issues, size_t maxErrors, const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr); ILexer::TPtr SqlLexer(const TString& query, NYql::TIssues& issues, const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr); + + /*[[deprecated]] Use SqlASTToYql(query, protoAst, hints, settings)*/ NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings); + NYql::TAstParseResult SqlASTToYql(const TString& query, const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings); + TVector SqlToAstStatements(const TString& query, const TTranslationSettings& settings, NYql::TWarningRules* warningRules = nullptr, ui16* actualSyntaxVersion = nullptr, TVector* stmtParseInfo = nullptr); diff --git a/ydb/library/yql/sql/v1/context.cpp b/ydb/library/yql/sql/v1/context.cpp index df5b4584d01a..7920ce356093 100644 --- a/ydb/library/yql/sql/v1/context.cpp +++ b/ydb/library/yql/sql/v1/context.cpp @@ -78,12 +78,14 @@ THashMap CTX_PRAGMA_MAYBE_FIELDS = { TContext::TContext(const NSQLTranslation::TTranslationSettings& settings, const NSQLTranslation::TSQLHints& hints, - TIssues& issues) + TIssues& issues, + const TString& query) : ClusterMapping(settings.ClusterMapping) , PathPrefix(settings.PathPrefix) , ClusterPathPrefixes(settings.ClusterPathPrefixes) , SQLHints(hints) , Settings(settings) + , Query(query) , Pool(new TMemoryPool(4096)) , Issues(issues) , IncrementMonCounterFunction(settings.IncrementCounter) diff --git a/ydb/library/yql/sql/v1/context.h b/ydb/library/yql/sql/v1/context.h index 70397f32c682..5e536939385c 100644 --- a/ydb/library/yql/sql/v1/context.h +++ b/ydb/library/yql/sql/v1/context.h @@ -87,7 +87,8 @@ namespace NSQLTranslationV1 { public: TContext(const NSQLTranslation::TTranslationSettings& settings, const NSQLTranslation::TSQLHints& hints, - NYql::TIssues& issues); + NYql::TIssues& issues, + const TString& query = {}); virtual ~TContext(); @@ -232,6 +233,7 @@ namespace NSQLTranslationV1 { THashMap> Variables; THashSet WeakVariables; NSQLTranslation::TTranslationSettings Settings; + const TString Query; std::unique_ptr Pool; NYql::TIssues& Issues; TMap UniversalAliases; @@ -318,6 +320,7 @@ namespace NSQLTranslationV1 { ui64 ParallelModeCount = 0; bool CompactNamedExprs = false; bool ValidateUnusedExprs = false; + TVector ForAllStatementsParts; }; class TColumnRefScope { diff --git a/ydb/library/yql/sql/v1/format/sql_format.cpp b/ydb/library/yql/sql/v1/format/sql_format.cpp index 9c03df7f5de0..4b2ac68ff4b2 100644 --- a/ydb/library/yql/sql/v1/format/sql_format.cpp +++ b/ydb/library/yql/sql/v1/format/sql_format.cpp @@ -35,13 +35,6 @@ TTokenIterator SkipWS(TTokenIterator curr, TTokenIterator end) { return curr; } -TTokenIterator SkipWSOrComment(TTokenIterator curr, TTokenIterator end) { - while (curr != end && (curr->Name == "WS" || curr->Name == "COMMENT")) { - ++curr; - } - return curr; -} - bool Validate(const TParsedTokenList& query, const TParsedTokenList& formattedQuery) { auto in = query.begin(); auto out = formattedQuery.begin(); @@ -73,139 +66,6 @@ bool Validate(const TParsedTokenList& query, const TParsedTokenList& formattedQu return in == inEnd && out == outEnd; } -enum EParenType { - Open, - Close, - None -}; - -using TAdvanceCallback = std::function; - -TTokenIterator SkipToNextBalanced(TTokenIterator begin, TTokenIterator end, const TAdvanceCallback& advance) { - i64 level = 0; - TTokenIterator curr = begin; - while (curr != end) { - switch (advance(curr, end)) { - case EParenType::Open: { - ++level; - break; - } - case EParenType::Close: { - --level; - if (level < 0) { - return end; - } else if (level == 0) { - return curr; - } - break; - } - case EParenType::None: - break; - } - } - return curr; -} - -TTokenIterator GetNextStatementBegin(TTokenIterator begin, TTokenIterator end) { - TAdvanceCallback advanceLambdaBody = [](TTokenIterator& curr, TTokenIterator end) -> EParenType { - Y_UNUSED(end); - if (curr->Name == "LBRACE_CURLY") { - ++curr; - return EParenType::Open; - } else if (curr->Name == "RBRACE_CURLY") { - ++curr; - return EParenType::Close; - } else { - ++curr; - return EParenType::None; - } - }; - - TAdvanceCallback advanceAction = [](TTokenIterator& curr, TTokenIterator end) -> EParenType { - auto tmp = curr; - if (curr->Name == "DEFINE") { - ++curr; - curr = SkipWSOrComment(curr, end); - if (curr != end && (curr->Name == "ACTION" || curr->Name == "SUBQUERY")) { - ++curr; - return EParenType::Open; - } - } else if (curr->Name == "END") { - ++curr; - curr = SkipWSOrComment(curr, end); - if (curr != end && curr->Name == "DEFINE") { - ++curr; - return EParenType::Close; - } - } - - curr = tmp; - ++curr; - return EParenType::None; - }; - - TAdvanceCallback advanceInlineAction = [](TTokenIterator& curr, TTokenIterator end) -> EParenType { - auto tmp = curr; - if (curr->Name == "DO") { - ++curr; - curr = SkipWSOrComment(curr, end); - if (curr != end && curr->Name == "BEGIN") { - ++curr; - return EParenType::Open; - } - } else if (curr->Name == "END") { - ++curr; - curr = SkipWSOrComment(curr, end); - if (curr != end && curr->Name == "DO") { - ++curr; - return EParenType::Close; - } - } - - curr = tmp; - ++curr; - return EParenType::None; - }; - - TTokenIterator curr = begin; - while (curr != end) { - bool matched = false; - for (auto cb : {advanceLambdaBody, advanceAction, advanceInlineAction}) { - TTokenIterator tmp = curr; - if (cb(tmp, end) == EParenType::Open) { - curr = SkipToNextBalanced(curr, end, cb); - matched = true; - if (curr == end) { - return curr; - } - } - } - if (matched) { - continue; - } - if (curr->Name == "SEMICOLON") { - ++curr; - break; - } - ++curr; - } - - return curr; -} - -void SplitByStatements(TTokenIterator begin, TTokenIterator end, TVector& output) { - output.clear(); - if (begin == end) { - return; - } - output.push_back(begin); - auto curr = begin; - while (curr != end) { - curr = GetNextStatementBegin(curr, end); - output.push_back(curr); - } -} - enum class EScope { Default, TypeName, @@ -2849,40 +2709,13 @@ class TSqlFormatter : public NSQLFormat::ISqlFormatter { } auto lexer = NSQLTranslationV1::MakeLexer(parsedSettings.AnsiLexer); - TParsedTokenList allTokens; - auto onNextToken = [&](NSQLTranslation::TParsedToken&& token) { - if (token.Name != "EOF") { - allTokens.push_back(token); - } - }; - - if (!lexer->Tokenize(query, "Query", onNextToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) { + TVector statements; + if (!NSQLTranslationV1::SplitQueryToStatements(query, lexer, statements, issues)) { return false; } - TVector statements; - SplitByStatements(allTokens.begin(), allTokens.end(), statements); TStringBuilder finalFormattedQuery; - for (size_t i = 1; i < statements.size(); ++i) { - TStringBuilder currentQueryBuilder; - for (auto it = statements[i - 1]; it != statements[i]; ++it) { - currentQueryBuilder << it->Content; - } - - TString currentQuery = currentQueryBuilder; - currentQuery = StripStringLeft(currentQuery); - bool isBlank = true; - for (auto c : currentQuery) { - if (c != ';') { - isBlank = false; - break; - } - }; - - if (isBlank) { - continue; - } - + for (const TString& currentQuery : statements) { TVector comments; TParsedTokenList parsedTokens, stmtTokens; bool hasTrailingComments = false; diff --git a/ydb/library/yql/sql/v1/lexer/lexer.cpp b/ydb/library/yql/sql/v1/lexer/lexer.cpp index 9f82c77b0b4a..c1b4b595547e 100644 --- a/ydb/library/yql/sql/v1/lexer/lexer.cpp +++ b/ydb/library/yql/sql/v1/lexer/lexer.cpp @@ -4,6 +4,10 @@ #include #include #include +#include + +#include +#include #if defined(_tsan_enabled_) #include @@ -62,4 +66,192 @@ NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi) { return NSQLTranslation::ILexer::TPtr(new TV1Lexer(ansi)); } +using NSQLTranslation::TParsedTokenList; +using TTokenIterator = TParsedTokenList::const_iterator; + +namespace { + +enum EParenType { + Open, + Close, + None +}; + +using TAdvanceCallback = std::function; + +TTokenIterator SkipWSOrComment(TTokenIterator curr, TTokenIterator end) { + while (curr != end && (curr->Name == "WS" || curr->Name == "COMMENT")) { + ++curr; + } + return curr; +} + +TTokenIterator SkipToNextBalanced(TTokenIterator begin, TTokenIterator end, const TAdvanceCallback& advance) { + i64 level = 0; + TTokenIterator curr = begin; + while (curr != end) { + switch (advance(curr, end)) { + case EParenType::Open: { + ++level; + break; + } + case EParenType::Close: { + --level; + if (level < 0) { + return end; + } else if (level == 0) { + return curr; + } + break; + } + case EParenType::None: + break; + } + } + return curr; +} + +TTokenIterator GetNextStatementBegin(TTokenIterator begin, TTokenIterator end) { + TAdvanceCallback advanceLambdaBody = [](TTokenIterator& curr, TTokenIterator end) -> EParenType { + Y_UNUSED(end); + if (curr->Name == "LBRACE_CURLY") { + ++curr; + return EParenType::Open; + } else if (curr->Name == "RBRACE_CURLY") { + ++curr; + return EParenType::Close; + } else { + ++curr; + return EParenType::None; + } + }; + + TAdvanceCallback advanceAction = [](TTokenIterator& curr, TTokenIterator end) -> EParenType { + auto tmp = curr; + if (curr->Name == "DEFINE") { + ++curr; + curr = SkipWSOrComment(curr, end); + if (curr != end && (curr->Name == "ACTION" || curr->Name == "SUBQUERY")) { + ++curr; + return EParenType::Open; + } + } else if (curr->Name == "END") { + ++curr; + curr = SkipWSOrComment(curr, end); + if (curr != end && curr->Name == "DEFINE") { + ++curr; + return EParenType::Close; + } + } + + curr = tmp; + ++curr; + return EParenType::None; + }; + + TAdvanceCallback advanceInlineAction = [](TTokenIterator& curr, TTokenIterator end) -> EParenType { + auto tmp = curr; + if (curr->Name == "DO") { + ++curr; + curr = SkipWSOrComment(curr, end); + if (curr != end && curr->Name == "BEGIN") { + ++curr; + return EParenType::Open; + } + } else if (curr->Name == "END") { + ++curr; + curr = SkipWSOrComment(curr, end); + if (curr != end && curr->Name == "DO") { + ++curr; + return EParenType::Close; + } + } + + curr = tmp; + ++curr; + return EParenType::None; + }; + + TTokenIterator curr = begin; + while (curr != end) { + bool matched = false; + for (auto cb : {advanceLambdaBody, advanceAction, advanceInlineAction}) { + TTokenIterator tmp = curr; + if (cb(tmp, end) == EParenType::Open) { + curr = SkipToNextBalanced(curr, end, cb); + matched = true; + if (curr == end) { + return curr; + } + } + } + if (matched) { + continue; + } + if (curr->Name == "SEMICOLON") { + ++curr; + break; + } + ++curr; + } + + return curr; +} + +void SplitByStatements(TTokenIterator begin, TTokenIterator end, TVector& output) { + output.clear(); + if (begin == end) { + return; + } + output.push_back(begin); + auto curr = begin; + while (curr != end) { + curr = GetNextStatementBegin(curr, end); + output.push_back(curr); + } +} + +} + +bool SplitQueryToStatements(const TString& query, NSQLTranslation::ILexer::TPtr& lexer, TVector& statements, NYql::TIssues& issues) { + TParsedTokenList allTokens; + auto onNextToken = [&](NSQLTranslation::TParsedToken&& token) { + if (token.Name != "EOF") { + allTokens.push_back(token); + } + }; + + if (!lexer->Tokenize(query, "Query", onNextToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) { + return false; + } + + TVector statementsTokens; + SplitByStatements(allTokens.begin(), allTokens.end(), statementsTokens); + + for (size_t i = 1; i < statementsTokens.size(); ++i) { + TStringBuilder currentQueryBuilder; + for (auto it = statementsTokens[i - 1]; it != statementsTokens[i]; ++it) { + currentQueryBuilder << it->Content; + } + TString statement = currentQueryBuilder; + statement = StripStringLeft(statement); + + bool isBlank = true; + for (auto c : statement) { + if (c != ';') { + isBlank = false; + break; + } + }; + + if (isBlank) { + continue; + } + + statements.push_back(statement); + } + + return true; +} + } // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/lexer/lexer.h b/ydb/library/yql/sql/v1/lexer/lexer.h index 312c1e207247..6fc5e5a7c464 100644 --- a/ydb/library/yql/sql/v1/lexer/lexer.h +++ b/ydb/library/yql/sql/v1/lexer/lexer.h @@ -6,4 +6,6 @@ namespace NSQLTranslationV1 { NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi); +bool SplitQueryToStatements(const TString& query, NSQLTranslation::ILexer::TPtr& lexer, + TVector& statements, NYql::TIssues& issues); } diff --git a/ydb/library/yql/sql/v1/object_processing.cpp b/ydb/library/yql/sql/v1/object_processing.cpp index 58f66df30b59..f6dcfa85d512 100644 --- a/ydb/library/yql/sql/v1/object_processing.cpp +++ b/ydb/library/yql/sql/v1/object_processing.cpp @@ -57,6 +57,26 @@ INode::TPtr TCreateObject::FillFeatures(INode::TPtr options) const { return options; } +namespace { + +bool InitFeatures(TContext& ctx, ISource* src, std::map& features) { + for (auto& [key, value] : features) { + if (value.HasNode() && !value.Build()->Init(ctx, src)) { + return false; + } + } + return true; +} + +} + +bool TCreateObject::DoInit(TContext& ctx, ISource* src) { + if (!InitFeatures(ctx, src, Features)) { + return false; + } + return TObjectProcessorImpl::DoInit(ctx, src); +} + TObjectOperatorContext::TObjectOperatorContext(TScopedStatePtr scoped) : Scoped(scoped) , ServiceId(Scoped->CurrService) diff --git a/ydb/library/yql/sql/v1/object_processing.h b/ydb/library/yql/sql/v1/object_processing.h index 4114235ee6dc..3bfe83d53591 100644 --- a/ydb/library/yql/sql/v1/object_processing.h +++ b/ydb/library/yql/sql/v1/object_processing.h @@ -55,6 +55,7 @@ class TCreateObject: public TObjectProcessorImpl { return Y(Q(Y(Q("mode"), Q(mode)))); } virtual INode::TPtr FillFeatures(INode::TPtr options) const override; + bool DoInit(TContext& ctx, ISource* src) override; public: TCreateObject(TPosition pos, const TString& objectId, const TString& typeId, bool existingOk, bool replaceIfExists, std::map&& features, std::set&& featuresToReset, const TObjectOperatorContext& context) diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index aacb8774e0a1..c45c58c90bcc 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -75,13 +75,21 @@ void SqlASTsToYqlsImpl(NYql::TAstParseResult& res, const std::vector<::NSQLv1Gen } } -NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, +NYql::TAstParseResult SqlASTToYql( + const google::protobuf::Message& protoAst, + const NSQLTranslation::TSQLHints& hints, + const NSQLTranslation::TTranslationSettings& settings) { + return SqlASTToYql("", protoAst, hints, settings); +} + +NYql::TAstParseResult SqlASTToYql(const TString& query, + const google::protobuf::Message& protoAst, const NSQLTranslation::TSQLHints& hints, const NSQLTranslation::TTranslationSettings& settings) { YQL_ENSURE(IsQueryMode(settings.Mode)); TAstParseResult res; - TContext ctx(settings, hints, res.Issues); + TContext ctx(settings, hints, res.Issues, query); SqlASTToYqlImpl(res, protoAst, ctx); res.ActualSyntaxType = NYql::ESyntaxType::YQLv1; return res; @@ -99,7 +107,7 @@ NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTra return res; } - TContext ctx(settings, hints, res.Issues); + TContext ctx(settings, hints, res.Issues, query); NSQLTranslation::TErrorCollectorOverIssues collector(res.Issues, settings.MaxErrors, settings.File); google::protobuf::Message* ast(SqlAST(query, queryName, collector, settings.AnsiLexer, settings.Arena)); @@ -140,22 +148,22 @@ bool NeedUseForAllStatements(const TRule_sql_stmt_core::AltCase& subquery) { case TRule_sql_stmt_core::kAltSqlStmtCore17: // do case TRule_sql_stmt_core::kAltSqlStmtCore19: // if case TRule_sql_stmt_core::kAltSqlStmtCore20: // for - case TRule_sql_stmt_core::kAltSqlStmtCore21: // values + case TRule_sql_stmt_core::kAltSqlStmtCore21: // values case TRule_sql_stmt_core::kAltSqlStmtCore22: // create user case TRule_sql_stmt_core::kAltSqlStmtCore23: // alter user case TRule_sql_stmt_core::kAltSqlStmtCore24: // create group - case TRule_sql_stmt_core::kAltSqlStmtCore25: // alter group - case TRule_sql_stmt_core::kAltSqlStmtCore26: // drop role - case TRule_sql_stmt_core::kAltSqlStmtCore27: // create object - case TRule_sql_stmt_core::kAltSqlStmtCore28: // alter object - case TRule_sql_stmt_core::kAltSqlStmtCore29: // drop object - case TRule_sql_stmt_core::kAltSqlStmtCore30: // create external data source - case TRule_sql_stmt_core::kAltSqlStmtCore31: // alter external data source - case TRule_sql_stmt_core::kAltSqlStmtCore32: // drop external data source - case TRule_sql_stmt_core::kAltSqlStmtCore33: // create replication - case TRule_sql_stmt_core::kAltSqlStmtCore34: // drop replication - case TRule_sql_stmt_core::kAltSqlStmtCore35: // create topic - case TRule_sql_stmt_core::kAltSqlStmtCore36: // alter topic + case TRule_sql_stmt_core::kAltSqlStmtCore25: // alter group + case TRule_sql_stmt_core::kAltSqlStmtCore26: // drop role + case TRule_sql_stmt_core::kAltSqlStmtCore27: // create object + case TRule_sql_stmt_core::kAltSqlStmtCore28: // alter object + case TRule_sql_stmt_core::kAltSqlStmtCore29: // drop object + case TRule_sql_stmt_core::kAltSqlStmtCore30: // create external data source + case TRule_sql_stmt_core::kAltSqlStmtCore31: // alter external data source + case TRule_sql_stmt_core::kAltSqlStmtCore32: // drop external data source + case TRule_sql_stmt_core::kAltSqlStmtCore33: // create replication + case TRule_sql_stmt_core::kAltSqlStmtCore34: // drop replication + case TRule_sql_stmt_core::kAltSqlStmtCore35: // create topic + case TRule_sql_stmt_core::kAltSqlStmtCore36: // alter topic case TRule_sql_stmt_core::kAltSqlStmtCore37: // drop topic case TRule_sql_stmt_core::kAltSqlStmtCore38: // grant permissions case TRule_sql_stmt_core::kAltSqlStmtCore39: // revoke permissions @@ -175,7 +183,7 @@ bool NeedUseForAllStatements(const TRule_sql_stmt_core::AltCase& subquery) { } } -TVector SqlToAstStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules, +TVector SqlToAstStatements(const TString& queryText, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules, TVector* stmtParseInfo) { TVector result; @@ -185,14 +193,14 @@ TVector SqlToAstStatements(const TString& query, const NS NSQLTranslation::TSQLHints hints; auto lexer = MakeLexer(settings.AnsiLexer); YQL_ENSURE(lexer); - if (!CollectSqlHints(*lexer, query, queryName, settings.File, hints, issues, settings.MaxErrors)) { + if (!CollectSqlHints(*lexer, queryText, queryName, settings.File, hints, issues, settings.MaxErrors)) { return result; } TContext ctx(settings, hints, issues); NSQLTranslation::TErrorCollectorOverIssues collector(issues, settings.MaxErrors, settings.File); - google::protobuf::Message* astProto(SqlAST(query, queryName, collector, settings.AnsiLexer, settings.Arena)); + google::protobuf::Message* astProto(SqlAST(queryText, queryName, collector, settings.AnsiLexer, settings.Arena)); if (astProto) { auto ast = static_cast(*astProto); const auto& query = ast.GetRule_sql_query(); @@ -203,7 +211,7 @@ TVector SqlToAstStatements(const TString& query, const NS if (NeedUseForAllStatements(statements.GetRule_sql_stmt2().GetRule_sql_stmt_core2().Alt_case())) { commonStates.push_back(statements.GetRule_sql_stmt2().GetRule_sql_stmt_core2()); } else { - TContext ctx(settings, hints, issues); + TContext ctx(settings, hints, issues, queryText); result.emplace_back(); if (stmtParseInfo) { stmtParseInfo->push_back({}); @@ -217,7 +225,7 @@ TVector SqlToAstStatements(const TString& query, const NS commonStates.push_back(block.GetRule_sql_stmt2().GetRule_sql_stmt_core2()); continue; } - TContext ctx(settings, hints, issues); + TContext ctx(settings, hints, issues, queryText); result.emplace_back(); if (stmtParseInfo) { stmtParseInfo->push_back({}); @@ -239,4 +247,28 @@ TVector SqlToAstStatements(const TString& query, const NS return result; } +bool SplitQueryToStatements(const TString& query, TVector& statements, NYql::TIssues& issues, + const NSQLTranslation::TTranslationSettings& settings) { + auto lexer = NSQLTranslationV1::MakeLexer(settings.AnsiLexer); + + TVector parts; + if (!SplitQueryToStatements(query, lexer, parts, issues)) { + return false; + } + + for (auto& currentQuery : parts) { + NYql::TIssues parserIssues; + auto message = NSQLTranslationV1::SqlAST(currentQuery, "Query", parserIssues, NSQLTranslation::SQL_MAX_PARSER_ERRORS, + settings.AnsiLexer, settings.Arena); + if (!message) { + // Skip empty statements + continue; + } + + statements.push_back(std::move(currentQuery)); + } + + return true; +} + } // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/sql.h b/ydb/library/yql/sql/v1/sql.h index 4b4ed87de59d..d433b239a1cd 100644 --- a/ydb/library/yql/sql/v1/sql.h +++ b/ydb/library/yql/sql/v1/sql.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -16,7 +17,16 @@ namespace NSQLTranslation { namespace NSQLTranslationV1 { NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules = nullptr); + + /*[[deprecated]] Use SqlASTToYql(query, protoAst, hints, settings)*/ NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const NSQLTranslation::TSQLHints& hints, const NSQLTranslation::TTranslationSettings& settings); + NYql::TAstParseResult SqlASTToYql(const TString& query, const google::protobuf::Message& protoAst, const NSQLTranslation::TSQLHints& hints, const NSQLTranslation::TTranslationSettings& settings); + TVector SqlToAstStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules, TVector* stmtParseInfo = nullptr); + bool NeedUseForAllStatements(const NSQLv1Generated::TRule_sql_stmt_core::AltCase& subquery); + + bool SplitQueryToStatements(const TString& query, TVector& statements, NYql::TIssues& issues, + const NSQLTranslation::TTranslationSettings& settings); + } // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/sql_query.cpp b/ydb/library/yql/sql/v1/sql_query.cpp index 40c1b6727204..e9496a915230 100644 --- a/ydb/library/yql/sql/v1/sql_query.cpp +++ b/ydb/library/yql/sql/v1/sql_query.cpp @@ -112,7 +112,7 @@ static bool AsyncReplicationAlterAction(std::map& settings, return AsyncReplicationSettings(settings, in.GetRule_alter_replication_set_setting1().GetRule_replication_settings3(), ctx, false); } -bool TSqlQuery::Statement(TVector& blocks, const TRule_sql_stmt_core& core) { +bool TSqlQuery::Statement(TVector& blocks, const TRule_sql_stmt_core& core, size_t statementNumber) { TString internalStatementName; TString humanStatementName; ParseStatementName(core, internalStatementName, humanStatementName); @@ -130,6 +130,10 @@ bool TSqlQuery::Statement(TVector& blocks, const TRule_sql_stmt_core& return false; } + if (NeedUseForAllStatements(altCase)) { + Ctx.ForAllStatementsParts.push_back(statementNumber); + } + switch (altCase) { case TRule_sql_stmt_core::kAltSqlStmtCore1: { bool success = false; @@ -3030,12 +3034,13 @@ TNodePtr TSqlQuery::Build(const TSQLv1ParserAST& ast) { Ctx.PopCurrentBlocks(); }; if (query.Alt_case() == TRule_sql_query::kAltSqlQuery1) { + size_t statementNumber = 0; const auto& statements = query.GetAlt_sql_query1().GetRule_sql_stmt_list1(); - if (!Statement(blocks, statements.GetRule_sql_stmt2().GetRule_sql_stmt_core2())) { + if (!Statement(blocks, statements.GetRule_sql_stmt2().GetRule_sql_stmt_core2(), statementNumber++)) { return nullptr; } for (auto block: statements.GetBlock3()) { - if (!Statement(blocks, block.GetRule_sql_stmt2().GetRule_sql_stmt_core2())) { + if (!Statement(blocks, block.GetRule_sql_stmt2().GetRule_sql_stmt_core2(), statementNumber++)) { return nullptr; } } @@ -3105,8 +3110,10 @@ TNodePtr TSqlQuery::Build(const std::vector<::NSQLv1Generated::TRule_sql_stmt_co Y_DEFER { Ctx.PopCurrentBlocks(); }; + + size_t statementNumber = 0; for (const auto& statement : statements) { - if (!Statement(blocks, statement)) { + if (!Statement(blocks, statement, statementNumber++)) { return nullptr; } } diff --git a/ydb/library/yql/sql/v1/sql_query.h b/ydb/library/yql/sql/v1/sql_query.h index 32dd9605cde8..4faa218bb86b 100644 --- a/ydb/library/yql/sql/v1/sql_query.h +++ b/ydb/library/yql/sql/v1/sql_query.h @@ -20,7 +20,7 @@ class TSqlQuery: public TSqlTranslation { TNodePtr Build(const TSQLv1ParserAST& ast); TNodePtr Build(const std::vector<::NSQLv1Generated::TRule_sql_stmt_core>& ast); - bool Statement(TVector& blocks, const TRule_sql_stmt_core& core); + bool Statement(TVector& blocks, const TRule_sql_stmt_core& core, size_t statementNumber); private: bool DeclareStatement(const TRule_declare_stmt& stmt); bool ExportStatement(const TRule_export_stmt& stmt); diff --git a/ydb/library/yql/sql/v1/sql_translation.cpp b/ydb/library/yql/sql/v1/sql_translation.cpp index 570dd98202d3..944faac6be79 100644 --- a/ydb/library/yql/sql/v1/sql_translation.cpp +++ b/ydb/library/yql/sql/v1/sql_translation.cpp @@ -19,6 +19,8 @@ namespace { using namespace NSQLTranslationV1; +using NSQLTranslation::ESqlMode; + template void VisitAllFields(const NProtoBuf::Message& msg, Callback& callback) { const auto* descr = msg.GetDescriptor(); @@ -54,67 +56,49 @@ TString CollectTokens(const TRule_select_stmt& selectStatement) { return tokenCollector.Tokens; } -bool RecreateContext( - TContext& ctx, const NSQLTranslation::TTranslationSettings& settings, const TString& recreationQuery -) { - if (!recreationQuery) { - return true; - } - const TString queryName = "context recreation query"; - - const auto* ast = NSQLTranslationV1::SqlAST( - recreationQuery, queryName, ctx.Issues, - settings.MaxErrors, settings.AnsiLexer, settings.Arena - ); - if (!ast) { +bool BuildContextRecreationQuery(TContext& context, TStringBuilder& query) { + TVector statements; + if (!SplitQueryToStatements(context.Query, statements, context.Issues, context.Settings)) { return false; } - TSqlQuery queryTranslator(ctx, ctx.Settings.Mode, true); - auto node = queryTranslator.Build(static_cast(*ast)); - - return node && node->Init(ctx, nullptr) && node->Translate(ctx); + for (size_t id : context.ForAllStatementsParts) { + query << statements[id] << '\n'; + } + return true; } -TNodePtr BuildViewSelect( - const TRule_select_stmt& selectStatement, - TContext& parentContext, - const TString& contextRecreationQuery -) { - TIssues issues; - TContext context(parentContext.Settings, {}, issues); - if (!RecreateContext(context, context.Settings, contextRecreationQuery)) { - parentContext.Issues.AddIssues(issues); - return nullptr; - } - issues.Clear(); +// ensures that the parsing mode is restored to the original value +class TModeGuard { + ESqlMode& Mode; + ESqlMode OriginalMode; - // Holds (among other things) subquery references. - // These references need to be passed to the parent context - // to be able to compile view queries with subqueries. - context.PushCurrentBlocks(&parentContext.GetCurrentBlocks()); +public: + TModeGuard(ESqlMode& mode, ESqlMode newMode) + : Mode(mode) + , OriginalMode(std::exchange(mode, newMode)) + {} - context.Settings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW; + ~TModeGuard() { + Mode = OriginalMode; + } +}; +TNodePtr BuildViewSelect(const TRule_select_stmt& selectStatement, TContext& context) { + TModeGuard guard(context.Settings.Mode, ESqlMode::LIMITED_VIEW); TSqlSelect selectTranslator(context, context.Settings.Mode); - TPosition pos = parentContext.Pos(); - auto source = selectTranslator.Build(selectStatement, pos); + auto position = context.Pos(); + auto source = selectTranslator.Build(selectStatement, position); if (!source) { - parentContext.Issues.AddIssues(issues); return nullptr; } - auto node = BuildSelectResult( - pos, - std::move(source), - false, - false, + return BuildSelectResult( + position, + source, + false, /* write result */ + false, /* in subquery */ context.Scoped ); - if (!node) { - parentContext.Issues.AddIssues(issues); - return nullptr; - } - return node; } } @@ -4192,13 +4176,15 @@ bool TSqlTranslation::DefineActionOrSubqueryBody(TSqlQuery& query, TBlocks& bloc Y_DEFER { Ctx.PopCurrentBlocks(); }; - if (!query.Statement(blocks, body.GetBlock2().GetRule_sql_stmt_core1())) { + + size_t statementNumber = 0; + if (!query.Statement(blocks, body.GetBlock2().GetRule_sql_stmt_core1(), statementNumber++)) { return false; } for (const auto& nestedStmtItem : body.GetBlock2().GetBlock2()) { const auto& nestedStmt = nestedStmtItem.GetRule_sql_stmt_core2(); - if (!query.Statement(blocks, nestedStmt)) { + if (!query.Statement(blocks, nestedStmt, statementNumber++)) { return false; } } @@ -4607,32 +4593,20 @@ bool TSqlTranslation::ParseViewQuery( std::map& features, const TRule_select_stmt& query ) { - TString queryText = CollectTokens(query); - TString contextRecreationQuery; - { - const auto& service = Ctx.Scoped->CurrService; - const auto& cluster = Ctx.Scoped->CurrCluster; - const auto effectivePathPrefix = Ctx.GetPrefixPath(service, cluster); - - // TO DO: capture all runtime pragmas in a similar fashion. - if (effectivePathPrefix != Ctx.Settings.PathPrefix) { - contextRecreationQuery = TStringBuilder() << "PRAGMA TablePathPrefix = \"" << effectivePathPrefix << "\";\n"; - } - - // TO DO: capture other compilation-affecting statements except USE. - if (cluster.GetLiteral() && *cluster.GetLiteral() != Ctx.Settings.DefaultCluster) { - contextRecreationQuery = TStringBuilder() << "USE " << *cluster.GetLiteral() << ";\n"; - } + TStringBuilder queryText; + if (!BuildContextRecreationQuery(Ctx, queryText)) { + return false; } - features["query_text"] = { Ctx.Pos(), contextRecreationQuery + queryText }; + queryText << CollectTokens(query); + features["query_text"] = { Ctx.Pos(), queryText }; - // AST is needed for ready-made validation of CREATE VIEW statement. - // Query is stored as plain text, not AST. - const auto viewSelect = BuildViewSelect(query, Ctx, contextRecreationQuery); + // The AST is needed solely for the validation of the CREATE VIEW statement. + // The final storage format for the query is a plain text, not an AST. + const auto viewSelect = BuildViewSelect(query, Ctx); if (!viewSelect) { return false; } - features["query_ast"] = {viewSelect, Ctx}; + features["query_ast"] = { viewSelect, Ctx }; return true; } diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index 1570fabfc4b7..4b0a7d9f8d94 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -6592,6 +6593,15 @@ Y_UNIT_TEST_SUITE(TViewSyntaxTest) { UNIT_ASSERT_C(res.Root, res.Issues.ToString()); } + Y_UNIT_TEST(CreateViewWithUdfs) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + CREATE VIEW TheView WITH (security_invoker = TRUE) AS SELECT "bbb" LIKE Unwrap("aaa"); + )" + ); + UNIT_ASSERT_C(res.Root, res.Issues.ToString()); + } + Y_UNIT_TEST(CreateViewIfNotExists) { constexpr const char* name = "TheView"; NYql::TAstParseResult res = SqlToYql(std::format(R"( @@ -7051,3 +7061,60 @@ Y_UNIT_TEST_SUITE(ResourcePoolClassifier) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); } } + +Y_UNIT_TEST_SUITE(QuerySplit) { + Y_UNIT_TEST(Simple) { + TString query = R"( + ; + -- Comment 1 + SELECT * From Input; -- Comment 2 + -- Comment 3 + $a = "a"; + + -- Comment 9 + ; + + -- Comment 10 + + -- Comment 8 + + $b = ($x) -> { + -- comment 4 + return /* Comment 5 */ $x; + -- Comment 6 + }; + + // Comment 7 + + + + )"; + + google::protobuf::Arena Arena; + + NSQLTranslation::TTranslationSettings settings; + settings.AnsiLexer = false; + settings.Arena = &Arena; + + TVector statements; + NYql::TIssues issues; + + UNIT_ASSERT(NSQLTranslationV1::SplitQueryToStatements(query, statements, issues, settings)); + + UNIT_ASSERT_VALUES_EQUAL(statements.size(), 3); + + UNIT_ASSERT_VALUES_EQUAL(statements[0], "-- Comment 1\n SELECT * From Input;"); + UNIT_ASSERT_VALUES_EQUAL(statements[1], R"(-- Comment 2 + -- Comment 3 + $a = "a";)"); + UNIT_ASSERT_VALUES_EQUAL(statements[2], R"(-- Comment 10 + + -- Comment 8 + + $b = ($x) -> { + -- comment 4 + return /* Comment 5 */ $x; + -- Comment 6 + };)"); + } +} diff --git a/ydb/library/yql/tools/sql2yql/sql2yql.cpp b/ydb/library/yql/tools/sql2yql/sql2yql.cpp index 8dbcd94a027d..4f2e1854f108 100644 --- a/ydb/library/yql/tools/sql2yql/sql2yql.cpp +++ b/ydb/library/yql/tools/sql2yql/sql2yql.cpp @@ -314,7 +314,7 @@ int BuildAST(int argc, char* argv[]) { NSQLTranslation::TSQLHints hints; auto lexer = SqlLexer(query, parseRes.Issues, settings); if (lexer && CollectSqlHints(*lexer, query, queryFile, settings.File, hints, parseRes.Issues, settings.MaxErrors)) { - parseRes = NSQLTranslation::SqlASTToYql(*ast, hints, settings); + parseRes = NSQLTranslation::SqlASTToYql(query, *ast, hints, settings); } } } else { From 492cb73d19db4e587309d4c3e29e483ed008d5ea Mon Sep 17 00:00:00 2001 From: Ilnaz Nizametdinov Date: Thu, 6 Mar 2025 17:52:46 +0300 Subject: [PATCH 27/29] 24-4: Add CleanupEC2MetadataClient() (#15382) --- .../run/kikimr_services_initializers.cpp | 27 +++---------- .../run/kikimr_services_initializers.h | 2 - ydb/core/driver_lib/run/run.cpp | 2 - ydb/core/driver_lib/run/ya.make | 10 ----- .../ut_schema/ut_columnshard_schema.cpp | 9 ++--- ydb/core/tx/columnshard/ut_schema/ya.make | 2 +- .../tx/schemeshard/ut_backup/ut_backup.cpp | 9 ++--- ydb/core/tx/schemeshard/ut_backup/ya.make | 2 +- .../tx/schemeshard/ut_export/ut_export.cpp | 9 ++--- ydb/core/tx/schemeshard/ut_export/ya.make | 2 +- .../ut_export_reboots_s3.cpp | 9 ++--- .../schemeshard/ut_export_reboots_s3/ya.make | 2 +- .../tx/schemeshard/ut_restore/ut_restore.cpp | 8 ++-- ydb/core/tx/schemeshard/ut_restore/ya.make | 2 +- ydb/core/util/aws.cpp | 39 +++++++++++++++++++ ydb/core/util/aws.h | 8 ++++ ydb/core/util/ya.make | 12 ++++++ ydb/core/wrappers/s3_wrapper_ut.cpp | 9 ++--- ydb/core/wrappers/ut/ya.make | 4 +- ydb/services/ydb/backup_ut/ya.make | 2 +- ydb/services/ydb/backup_ut/ydb_backup_ut.cpp | 8 ++-- 21 files changed, 93 insertions(+), 84 deletions(-) create mode 100644 ydb/core/util/aws.cpp create mode 100644 ydb/core/util/aws.h diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index e5ffcc00263a..6d68ab94bc87 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -155,6 +155,7 @@ #include #include +#include #include #include #include @@ -244,32 +245,17 @@ #include -#ifndef KIKIMR_DISABLE_S3_OPS -#include -#endif - -namespace { +namespace NKikimr::NKikimrServicesInitializers { -#ifndef KIKIMR_DISABLE_S3_OPS struct TAwsApiGuard { TAwsApiGuard() { - Aws::InitAPI(Options); + InitAwsAPI(); } ~TAwsApiGuard() { - Aws::ShutdownAPI(Options); + ShutdownAwsAPI(); } - -private: - Aws::SDKOptions Options; }; -#endif - -} - -namespace NKikimr { - -namespace NKikimrServicesInitializers { ui32 TFederatedQueryInitializer::IcPort = 0; @@ -2884,7 +2870,6 @@ void TGraphServiceInitializer::InitializeServices(NActors::TActorSystemSetup* se TActorSetupCmd(NGraph::CreateGraphService(appData->TenantName), TMailboxType::HTSwap, appData->UserPoolId)); } -#ifndef KIKIMR_DISABLE_S3_OPS TAwsApiInitializer::TAwsApiInitializer(IGlobalObjectStorage& globalObjects) : GlobalObjects(globalObjects) { @@ -2895,7 +2880,5 @@ void TAwsApiInitializer::InitializeServices(NActors::TActorSystemSetup* setup, c Y_UNUSED(appData); GlobalObjects.AddGlobalObject(std::make_shared()); } -#endif -} // namespace NKikimrServicesInitializers -} // namespace NKikimr +} // namespace NKikimr::NKikimrServicesInitializers diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 04f30522186f..fa04ef57e038 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -618,7 +618,6 @@ class TGraphServiceInitializer : public IKikimrServicesInitializer { void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; -#ifndef KIKIMR_DISABLE_S3_OPS class TAwsApiInitializer : public IServiceInitializer { IGlobalObjectStorage& GlobalObjects; @@ -627,7 +626,6 @@ class TAwsApiInitializer : public IServiceInitializer { void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; -#endif } // namespace NKikimrServicesInitializers } // namespace NKikimr diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index fbf5397b2080..11c490865330 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -1685,11 +1685,9 @@ TIntrusivePtr TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TGraphServiceInitializer(runConfig)); } -#ifndef KIKIMR_DISABLE_S3_OPS if (serviceMask.EnableAwsService) { sil->AddServiceInitializer(new TAwsApiInitializer(*this)); } -#endif return sil; } diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index e479bf5b7b7e..730075b59657 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -1,15 +1,5 @@ LIBRARY(run) -IF (OS_WINDOWS) - CFLAGS( - -DKIKIMR_DISABLE_S3_OPS - ) -ELSE() - PEERDIR( - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core - ) -ENDIF() - SRCS( auto_config_initializer.cpp config.cpp diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp index deb7be3d89e9..6237e45d159b 100644 --- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -19,8 +20,6 @@ #include #include -#include - namespace NKikimr { using namespace NTxUT; @@ -35,14 +34,12 @@ enum class EInitialEviction { namespace { -Aws::SDKOptions Options; - Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); + NKikimr::InitAwsAPI(); } Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); + NKikimr::ShutdownAwsAPI(); } static const std::vector testYdbSchema = TTestSchema::YdbSchema(); diff --git a/ydb/core/tx/columnshard/ut_schema/ya.make b/ydb/core/tx/columnshard/ut_schema/ya.make index d67c0d2ad5b8..cccbdb15ed0e 100644 --- a/ydb/core/tx/columnshard/ut_schema/ya.make +++ b/ydb/core/tx/columnshard/ut_schema/ya.make @@ -18,13 +18,13 @@ PEERDIR( library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/testlib/default ydb/core/tx/columnshard/hooks/abstract ydb/core/tx/columnshard/hooks/testing ydb/core/tx/columnshard/test_helper ydb/services/metadata ydb/core/tx + ydb/core/util ydb/public/lib/yson_value ) diff --git a/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp b/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp index 771b929051c0..e2d43b4a7b58 100644 --- a/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp +++ b/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp @@ -1,6 +1,7 @@ #include "ut_helpers/ut_backup_restore_common.h" #include +#include #include #include @@ -9,21 +10,17 @@ #include -#include - using namespace NSchemeShardUT_Private; using namespace NKikimr::NWrappers::NTestHelpers; namespace { -Aws::SDKOptions Options; - Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); + NKikimr::InitAwsAPI(); } Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); + NKikimr::ShutdownAwsAPI(); } } diff --git a/ydb/core/tx/schemeshard/ut_backup/ya.make b/ydb/core/tx/schemeshard/ut_backup/ya.make index aac9bc5f9334..abd09f4dddc3 100644 --- a/ydb/core/tx/schemeshard/ut_backup/ya.make +++ b/ydb/core/tx/schemeshard/ut_backup/ya.make @@ -20,10 +20,10 @@ IF (NOT OS_WINDOWS) library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/testlib/default ydb/core/tx ydb/core/tx/schemeshard/ut_helpers + ydb/core/util ydb/core/wrappers/ut_helpers ydb/library/yql/public/udf/service/exception_policy ) diff --git a/ydb/core/tx/schemeshard/ut_export/ut_export.cpp b/ydb/core/tx/schemeshard/ut_export/ut_export.cpp index 8d41d7bab562..d739bd7cbc6f 100644 --- a/ydb/core/tx/schemeshard/ut_export/ut_export.cpp +++ b/ydb/core/tx/schemeshard/ut_export/ut_export.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -15,21 +16,17 @@ #include -#include - using namespace NSchemeShardUT_Private; using namespace NKikimr::NWrappers::NTestHelpers; namespace { - Aws::SDKOptions Options; - Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); + NKikimr::InitAwsAPI(); } Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); + NKikimr::ShutdownAwsAPI(); } void Run(TTestBasicRuntime& runtime, TTestEnv& env, const TVector& tables, const TString& request, diff --git a/ydb/core/tx/schemeshard/ut_export/ya.make b/ydb/core/tx/schemeshard/ut_export/ya.make index c62dc9ea8ebc..bd9be54e7007 100644 --- a/ydb/core/tx/schemeshard/ut_export/ya.make +++ b/ydb/core/tx/schemeshard/ut_export/ya.make @@ -20,10 +20,10 @@ IF (NOT OS_WINDOWS) library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/testlib/default ydb/core/tx ydb/core/tx/schemeshard/ut_helpers + ydb/core/util ydb/core/wrappers/ut_helpers ydb/library/yql/public/udf/service/exception_policy ) diff --git a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp index 97e34a6d3fa4..a08cf8d44eba 100644 --- a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp +++ b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp @@ -1,27 +1,24 @@ #include #include +#include #include #include #include -#include - using namespace NSchemeShardUT_Private; using namespace NSchemeShardUT_Private::NExportReboots; using namespace NKikimr::NWrappers::NTestHelpers; namespace { -Aws::SDKOptions Options; - Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); + NKikimr::InitAwsAPI(); } Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); + NKikimr::ShutdownAwsAPI(); } } diff --git a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make index caf4fb7de362..ef80b381004d 100644 --- a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make +++ b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make @@ -19,10 +19,10 @@ PEERDIR( library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/testlib/default ydb/core/tx ydb/core/tx/schemeshard/ut_helpers + ydb/core/util ydb/core/wrappers/ut_helpers ydb/library/yql/public/udf/service/exception_policy ) diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index fbc76affe9e7..15b6cb82e6fe 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -22,7 +23,6 @@ #include -#include #include #include #include @@ -41,14 +41,12 @@ using namespace NKikimr::NWrappers::NTestHelpers; namespace { - Aws::SDKOptions Options; - Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); + NKikimr::InitAwsAPI(); } Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); + NKikimr::ShutdownAwsAPI(); } const TString EmptyYsonStr = R"([[[[];%false]]])"; diff --git a/ydb/core/tx/schemeshard/ut_restore/ya.make b/ydb/core/tx/schemeshard/ut_restore/ya.make index d514b36b49ee..1ebba95cca66 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ya.make +++ b/ydb/core/tx/schemeshard/ut_restore/ya.make @@ -14,11 +14,11 @@ ELSE() ENDIF() PEERDIR( - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core contrib/libs/double-conversion library/cpp/string_utils/quote ydb/core/kqp/ut/common ydb/core/tx/schemeshard/ut_helpers + ydb/core/util ydb/core/wrappers/ut_helpers ydb/core/ydb_convert ydb/library/yql/sql/pg diff --git a/ydb/core/util/aws.cpp b/ydb/core/util/aws.cpp new file mode 100644 index 000000000000..9346a824cde6 --- /dev/null +++ b/ydb/core/util/aws.cpp @@ -0,0 +1,39 @@ +#include "aws.h" + +#ifndef KIKIMR_DISABLE_S3_OPS + +#include +#include +#include + +Aws::SDKOptions MakeDefaultOptions() { + static Aws::SDKOptions options; + options.httpOptions.initAndCleanupCurl = false; + return options; +} + +namespace NKikimr { + +void InitAwsAPI() { + curl_global_init(CURL_GLOBAL_ALL); + Aws::InitAPI(MakeDefaultOptions()); + Aws::Internal::CleanupEC2MetadataClient(); // speeds up config construction +} + +void ShutdownAwsAPI() { + Aws::ShutdownAPI(MakeDefaultOptions()); + curl_global_cleanup(); +} + +} // NKikimr + +#else + +namespace NKikimr { + +void InitAwsAPI() {} +void ShutdownAwsAPI() {} + +} // NKikimr + +#endif diff --git a/ydb/core/util/aws.h b/ydb/core/util/aws.h new file mode 100644 index 000000000000..fd85f60845f7 --- /dev/null +++ b/ydb/core/util/aws.h @@ -0,0 +1,8 @@ +#pragma once + +namespace NKikimr { + +void InitAwsAPI(); +void ShutdownAwsAPI(); + +} diff --git a/ydb/core/util/ya.make b/ydb/core/util/ya.make index 5067894348be..f8669e2fbd7f 100644 --- a/ydb/core/util/ya.make +++ b/ydb/core/util/ya.make @@ -3,6 +3,7 @@ LIBRARY() SRCS( activeactors.h address_classifier.cpp + aws.cpp backoff.cpp cache.cpp cache.h @@ -81,6 +82,17 @@ PEERDIR( ydb/library/yverify_stream ) +IF (OS_WINDOWS) + CFLAGS( + -DKIKIMR_DISABLE_S3_OPS + ) +ELSE() + PEERDIR( + contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core + contrib/libs/curl + ) +ENDIF() + END() RECURSE_FOR_TESTS( diff --git a/ydb/core/wrappers/s3_wrapper_ut.cpp b/ydb/core/wrappers/s3_wrapper_ut.cpp index b8b0e185cde4..a165aa916827 100644 --- a/ydb/core/wrappers/s3_wrapper_ut.cpp +++ b/ydb/core/wrappers/s3_wrapper_ut.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -13,8 +14,6 @@ #include -#include - using namespace NActors; using namespace NKikimr; using namespace NKikimr::NWrappers; @@ -22,14 +21,12 @@ using namespace Aws::S3::Model; namespace { -Aws::SDKOptions Options; - Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); + NKikimr::InitAwsAPI(); } Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); + NKikimr::ShutdownAwsAPI(); } } diff --git a/ydb/core/wrappers/ut/ya.make b/ydb/core/wrappers/ut/ya.make index 212e65cfc54f..85cdc31da258 100644 --- a/ydb/core/wrappers/ut/ya.make +++ b/ydb/core/wrappers/ut/ya.make @@ -9,11 +9,11 @@ IF (NOT OS_WINDOWS) ydb/library/actors/core library/cpp/digest/md5 library/cpp/testing/unittest - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/protos ydb/core/testlib/basics/default - ydb/library/yql/minikql/comp_nodes/llvm14 + ydb/core/util ydb/core/wrappers/ut_helpers + ydb/library/yql/minikql/comp_nodes/llvm14 ) SRCS( s3_wrapper_ut.cpp diff --git a/ydb/services/ydb/backup_ut/ya.make b/ydb/services/ydb/backup_ut/ya.make index 39732a69bd9c..f77495d38138 100644 --- a/ydb/services/ydb/backup_ut/ya.make +++ b/ydb/services/ydb/backup_ut/ya.make @@ -15,6 +15,7 @@ SRCS( PEERDIR( ydb/core/testlib/default + ydb/core/util ydb/core/wrappers/ut_helpers ydb/public/lib/ydb_cli/dump ydb/public/sdk/cpp/client/ydb_export @@ -24,7 +25,6 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_table ydb/public/sdk/cpp/client/ydb_value ydb/library/backup - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ) YQL_LAST_ABI_VERSION() diff --git a/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp b/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp index f66a08aff279..5575f56133de 100644 --- a/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp +++ b/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp @@ -1,5 +1,6 @@ #include "ydb_common_ut.h" +#include #include #include @@ -15,7 +16,6 @@ #include #include -#include #include using namespace NYdb; @@ -397,14 +397,12 @@ Y_UNIT_TEST_SUITE(BackupRestore) { Y_UNIT_TEST_SUITE(BackupRestoreS3) { - Aws::SDKOptions Options; - Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); + NKikimr::InitAwsAPI(); } Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); + NKikimr::ShutdownAwsAPI(); } using NKikimr::NWrappers::NTestHelpers::TS3Mock; From 65e857bcfd0c07d37b5fd449f4e66f22721e915b Mon Sep 17 00:00:00 2001 From: Andrey Molotkov Date: Tue, 4 Mar 2025 10:08:21 +0300 Subject: [PATCH 28/29] Refresh ldap token with error (#15182) --- .../ldap_auth_provider/ldap_auth_provider.cpp | 3 + .../ldap_auth_provider_linux.cpp | 5 ++ .../ldap_auth_provider_ut.cpp | 67 +++++++++++++++++++ .../ldap_auth_provider_win.cpp | 5 ++ ydb/core/security/ticket_parser_impl.h | 60 ++++++++++------- ydb/core/security/ticket_parser_ut.cpp | 66 ++++++++++++++++++ .../service_mocks/ldap_mock/ldap_defines.h | 1 + 7 files changed, 181 insertions(+), 26 deletions(-) diff --git a/ydb/core/security/ldap_auth_provider/ldap_auth_provider.cpp b/ydb/core/security/ldap_auth_provider/ldap_auth_provider.cpp index 0b950592df03..3ff313f48400 100644 --- a/ydb/core/security/ldap_auth_provider/ldap_auth_provider.cpp +++ b/ydb/core/security/ldap_auth_provider/ldap_auth_provider.cpp @@ -321,6 +321,7 @@ class TLdapAuthProvider : public NActors::TActorBootstrapped response.Status = NKikimrLdap::ErrorToStatus(result); response.Error = {.Message = ERROR_MESSAGE, .LogMessage = logErrorMessage, .Retryable = NKikimrLdap::IsRetryableError(result)}; LDAP_LOG_D(logErrorMessage); + NKikimrLdap::MsgFree(searchMessage); return response; } const int countEntries = NKikimrLdap::CountEntries(request.Ld, searchMessage); @@ -357,6 +358,7 @@ class TLdapAuthProvider : public NActors::TActorBootstrapped LDAPMessage* searchMessage = nullptr; int result = NKikimrLdap::Search(ld, Settings.GetBaseDn(), NKikimrLdap::EScope::SUBTREE, filter, NKikimrLdap::noAttributes, 0, &searchMessage); if (!NKikimrLdap::IsSuccess(result)) { + NKikimrLdap::MsgFree(searchMessage); return {}; } const int countEntries = NKikimrLdap::CountEntries(ld, searchMessage); @@ -403,6 +405,7 @@ class TLdapAuthProvider : public NActors::TActorBootstrapped LDAPMessage* searchMessage = nullptr; int result = NKikimrLdap::Search(ld, Settings.GetBaseDn(), NKikimrLdap::EScope::SUBTREE, filter, RequestedAttributes, 0, &searchMessage); if (!NKikimrLdap::IsSuccess(result)) { + NKikimrLdap::MsgFree(searchMessage); return; } if (NKikimrLdap::CountEntries(ld, searchMessage) == 0) { diff --git a/ydb/core/security/ldap_auth_provider/ldap_auth_provider_linux.cpp b/ydb/core/security/ldap_auth_provider/ldap_auth_provider_linux.cpp index 4125182f3d74..d514b5f253ca 100644 --- a/ydb/core/security/ldap_auth_provider/ldap_auth_provider_linux.cpp +++ b/ydb/core/security/ldap_auth_provider/ldap_auth_provider_linux.cpp @@ -158,6 +158,11 @@ NKikimr::TEvLdapAuthProvider::EStatus ErrorToStatus(int err) { bool IsRetryableError(int error) { switch (error) { case LDAP_SERVER_DOWN: + case LDAP_TIMEOUT: + case LDAP_CONNECT_ERROR: + case LDAP_BUSY: + case LDAP_UNAVAILABLE: + case LDAP_ADMINLIMIT_EXCEEDED: return true; } return false; diff --git a/ydb/core/security/ldap_auth_provider/ldap_auth_provider_ut.cpp b/ydb/core/security/ldap_auth_provider/ldap_auth_provider_ut.cpp index bfb5877bcf6d..db2019d2ce39 100644 --- a/ydb/core/security/ldap_auth_provider/ldap_auth_provider_ut.cpp +++ b/ydb/core/security/ldap_auth_provider/ldap_auth_provider_ut.cpp @@ -1168,6 +1168,61 @@ void CheckRequiredLdapSettings(std::functionAllocateEdgeActor(); + runtime->Send(new IEventHandle(MakeTicketParserID(), sender, new TEvTicketParser::TEvAuthorizeTicket(loginResponse.Token)), 0); + TAutoPtr handle; + TEvTicketParser::TEvAuthorizeTicketResult* ticketParserResult = runtime->GrabEdgeEvent(handle); + + // Server is busy, return retryable error + UNIT_ASSERT_C(!ticketParserResult->Error.empty(), "Expected return error message"); + UNIT_ASSERT(ticketParserResult->Token == nullptr); + UNIT_ASSERT_STRINGS_EQUAL(ticketParserResult->Error.Message, "Could not login via LDAP"); + UNIT_ASSERT_EQUAL(ticketParserResult->Error.Retryable, true); + + Sleep(TDuration::Seconds(3)); + ldapServer.UpdateResponses(); + Sleep(TDuration::Seconds(7)); + + runtime->Send(new IEventHandle(MakeTicketParserID(), sender, new TEvTicketParser::TEvAuthorizeTicket(loginResponse.Token)), 0); + ticketParserResult = runtime->GrabEdgeEvent(handle); + + // After refresh ticket, server return success + UNIT_ASSERT_C(ticketParserResult->Error.empty(), ticketParserResult->Error); + UNIT_ASSERT(ticketParserResult->Token != nullptr); + const TString ldapDomain = "@ldap"; + UNIT_ASSERT_VALUES_EQUAL(ticketParserResult->Token->GetUserSID(), login + ldapDomain); + const auto& fetchedGroups = ticketParserResult->Token->GetGroupSIDs(); + THashSet groups(fetchedGroups.begin(), fetchedGroups.end()); + + THashSet expectedGroups = TCorrectLdapResponse::GetAllGroups(ldapDomain); + expectedGroups.insert("all-users@well-known"); + + UNIT_ASSERT_VALUES_EQUAL(fetchedGroups.size(), expectedGroups.size()); + for (const auto& expectedGroup : expectedGroups) { + UNIT_ASSERT_C(groups.contains(expectedGroup), "Can not find " + expectedGroup); + } + + ldapServer.Stop(); + } + Y_UNIT_TEST_SUITE(LdapAuthProviderTest) { Y_UNIT_TEST(LdapServerIsUnavailable) { CheckRequiredLdapSettings(InitLdapSettingsWithUnavailableHost, "Could not login via LDAP", ESecurityConnectionType::START_TLS); @@ -1246,6 +1301,10 @@ Y_UNIT_TEST_SUITE(LdapAuthProviderTest_LdapsScheme) { Y_UNIT_TEST(LdapRefreshRemoveUserBad) { LdapRefreshRemoveUserBad(ESecurityConnectionType::LDAPS_SCHEME); } + + Y_UNIT_TEST(LdapRefreshGroupsInfoWithError) { + LdapRefreshGroupsInfoWithError(ESecurityConnectionType::LDAPS_SCHEME); + } } Y_UNIT_TEST_SUITE(LdapAuthProviderTest_StartTls) { @@ -1304,6 +1363,10 @@ Y_UNIT_TEST_SUITE(LdapAuthProviderTest_StartTls) { Y_UNIT_TEST(LdapRefreshRemoveUserBad) { LdapRefreshRemoveUserBad(ESecurityConnectionType::START_TLS); } + + Y_UNIT_TEST(LdapRefreshGroupsInfoWithError) { + LdapRefreshGroupsInfoWithError(ESecurityConnectionType::START_TLS); + } } Y_UNIT_TEST_SUITE(LdapAuthProviderTest_nonSecure) { @@ -1362,6 +1425,10 @@ Y_UNIT_TEST_SUITE(LdapAuthProviderTest_nonSecure) { Y_UNIT_TEST(LdapRefreshRemoveUserBad) { LdapRefreshRemoveUserBad(ESecurityConnectionType::NON_SECURE); } + + Y_UNIT_TEST(LdapRefreshGroupsInfoWithError) { + LdapRefreshGroupsInfoWithError(ESecurityConnectionType::NON_SECURE); + } } } // NKikimr diff --git a/ydb/core/security/ldap_auth_provider/ldap_auth_provider_win.cpp b/ydb/core/security/ldap_auth_provider/ldap_auth_provider_win.cpp index d1fc38a449ce..f84d80cbe518 100644 --- a/ydb/core/security/ldap_auth_provider/ldap_auth_provider_win.cpp +++ b/ydb/core/security/ldap_auth_provider/ldap_auth_provider_win.cpp @@ -147,6 +147,11 @@ NKikimr::TEvLdapAuthProvider::EStatus ErrorToStatus(int err) { bool IsRetryableError(int error) { switch (error) { case LDAP_SERVER_DOWN: + case LDAP_TIMEOUT: + case LDAP_CONNECT_ERROR: + case LDAP_BUSY: + case LDAP_UNAVAILABLE: + case LDAP_ADMIN_LIMIT_EXCEEDED: return true; } return false; diff --git a/ydb/core/security/ticket_parser_impl.h b/ydb/core/security/ticket_parser_impl.h index 1872aadf8c0f..5fe9fc2c9bc8 100644 --- a/ydb/core/security/ticket_parser_impl.h +++ b/ydb/core/security/ticket_parser_impl.h @@ -1741,12 +1741,16 @@ class TTicketParserImpl : public TActorBootstrapped { void SetError(const TString& key, TTokenRecord& record, const TEvTicketParser::TError& error) { record.Error = error; TInstant now = TlsActivationContext->Now(); + TStringBuilder errorLogMessage; + if (error.HasLogMessage()) { + errorLogMessage << " (" << error.LogMessage << ")"; + } if (record.Error.Retryable) { record.ExpireTime = GetExpireTime(record, now); record.SetErrorRefreshTime(this, now); CounterTicketsErrorsRetryable->Inc(); BLOG_D("Ticket " << record.GetMaskedTicket() << " (" - << record.PeerName << ") has now retryable error message '" << error.Message << "'"); + << record.PeerName << ") has now retryable error message '" << error.Message << errorLogMessage << "'"); if (record.RefreshRetryableErrorImmediately) { record.RefreshRetryableErrorImmediately = false; GetDerived()->CanRefreshTicket(key, record); @@ -1759,7 +1763,7 @@ class TTicketParserImpl : public TActorBootstrapped { record.SetOkRefreshTime(this, now); CounterTicketsErrorsPermanent->Inc(); BLOG_D("Ticket " << record.GetMaskedTicket() << " (" - << record.PeerName << ") has now permanent error message '" << error.Message << "'"); + << record.PeerName << ") has now permanent error message '" << error.Message << errorLogMessage << "'"); } CounterTicketsErrors->Inc(); record.IsLowAccessServiceRequestPriority = true; @@ -1841,7 +1845,7 @@ class TTicketParserImpl : public TActorBootstrapped { template bool CanRefreshLoginTicket(const TTokenRecord& record) { - return record.TokenType == TDerived::ETokenType::Login && record.Error.empty(); + return record.TokenType == TDerived::ETokenType::Login && record.Error.Retryable; } template @@ -1862,30 +1866,34 @@ class TTicketParserImpl : public TActorBootstrapped { template bool RefreshLoginTicket(const TString& key, TTokenRecord& record) { - GetDerived()->ResetTokenRecord(record); - const TString userSID = record.GetToken()->GetUserSID(); - if (record.IsExternalAuthEnabled()) { - return RefreshTicketViaExternalAuthProvider(key, record); - } - const TString& database = Config.GetDomainLoginOnly() ? DomainName : record.Database; - auto itLoginProvider = LoginProviders.find(database); - if (itLoginProvider == LoginProviders.end()) { - return false; - } - NLogin::TLoginProvider& loginProvider(itLoginProvider->second); - if (loginProvider.CheckUserExists(userSID)) { - const std::vector providerGroups = loginProvider.GetGroupsMembership(userSID); - const TVector groups(providerGroups.begin(), providerGroups.end()); - SetToken(key, record, new NACLib::TUserToken({ - .OriginalUserToken = record.Ticket, - .UserSID = userSID, - .GroupSIDs = groups, - .AuthType = record.GetAuthType() - })); - } else { - SetError(key, record, {.Message = "User not found", .Retryable = false}); + if (record.Error.empty()) { + GetDerived()->ResetTokenRecord(record); + const TString userSID = record.GetToken()->GetUserSID(); + if (record.IsExternalAuthEnabled()) { + return RefreshTicketViaExternalAuthProvider(key, record); + } + const TString& database = Config.GetDomainLoginOnly() ? DomainName : record.Database; + auto itLoginProvider = LoginProviders.find(database); + if (itLoginProvider == LoginProviders.end()) { + return false; + } + NLogin::TLoginProvider& loginProvider(itLoginProvider->second); + if (loginProvider.CheckUserExists(userSID)) { + const std::vector providerGroups = loginProvider.GetGroupsMembership(userSID); + const TVector groups(providerGroups.begin(), providerGroups.end()); + SetToken(key, record, new NACLib::TUserToken({ + .OriginalUserToken = record.Ticket, + .UserSID = userSID, + .GroupSIDs = groups, + .AuthType = record.GetAuthType() + })); + } else { + SetError(key, record, {.Message = "User not found", .Retryable = false}); + } + return true; } - return true; + GetDerived()->ResetTokenRecord(record); + return CanInitLoginToken(key, record); } template diff --git a/ydb/core/security/ticket_parser_ut.cpp b/ydb/core/security/ticket_parser_ut.cpp index efdb20441ee4..90474c6824b3 100644 --- a/ydb/core/security/ticket_parser_ut.cpp +++ b/ydb/core/security/ticket_parser_ut.cpp @@ -278,6 +278,72 @@ Y_UNIT_TEST_SUITE(TTicketParserTest) { UNIT_ASSERT_VALUES_EQUAL(result->Token->GetGroupSIDs().size(), 3); } + Y_UNIT_TEST(LoginRefreshGroupsWithError) { + using namespace Tests; + TPortManager tp; + ui16 kikimrPort = tp.GetPort(2134); + ui16 grpcPort = tp.GetPort(2135); + NKikimrProto::TAuthConfig authConfig; + authConfig.SetUseBlackBox(false); + authConfig.SetUseLoginProvider(true); + authConfig.SetRefreshTime("5s"); + auto settings = TServerSettings(kikimrPort, authConfig); + settings.SetDomainName("Root"); + settings.CreateTicketParser = NKikimr::CreateTicketParser; + TServer server(settings); + server.EnableGRpc(grpcPort); + server.GetRuntime()->SetLogPriority(NKikimrServices::TICKET_PARSER, NLog::PRI_TRACE); + server.GetRuntime()->SetLogPriority(NKikimrServices::GRPC_CLIENT, NLog::PRI_TRACE); + TClient client(settings); + NClient::TKikimr kikimr(client.GetClientConfig()); + client.InitRootScheme(); + TTestActorRuntime* runtime = server.GetRuntime(); + + NLogin::TLoginProvider provider; + + provider.Audience = "/Root"; + provider.RotateKeys(); + + TActorId sender = runtime->AllocateEdgeActor(); + + provider.CreateGroup({.Group = "group1"}); + provider.CreateUser({.User = "user1", .Password = "password1"}); + provider.AddGroupMembership({.Group = "group1", .Member = "user1"}); + + NLogin::TLoginProvider emptyProvider; + emptyProvider.Audience = "/Root"; + + runtime->Send(new IEventHandle(MakeTicketParserID(), sender, new TEvTicketParser::TEvUpdateLoginSecurityState(emptyProvider.GetSecurityState())), 0); + + auto loginResponse = provider.LoginUser({.User = "user1", .Password = "password1"}); + + UNIT_ASSERT_VALUES_EQUAL(loginResponse.Error, ""); + + runtime->Send(new IEventHandle(MakeTicketParserID(), sender, new TEvTicketParser::TEvAuthorizeTicket(loginResponse.Token)), 0); + + TAutoPtr handle; + + TEvTicketParser::TEvAuthorizeTicketResult* result = runtime->GrabEdgeEvent(handle); + UNIT_ASSERT_C(!result->Error.empty(), "Expected return error message"); + UNIT_ASSERT(result->Token == nullptr); + UNIT_ASSERT_STRINGS_EQUAL(result->Error.Message, "Security state is empty"); + UNIT_ASSERT_EQUAL(result->Error.Retryable, true); + + Sleep(TDuration::Seconds(3)); + runtime->Send(new IEventHandle(MakeTicketParserID(), sender, new TEvTicketParser::TEvUpdateLoginSecurityState(provider.GetSecurityState())), 0); + Sleep(TDuration::Seconds(7)); + + runtime->Send(new IEventHandle(MakeTicketParserID(), sender, new TEvTicketParser::TEvAuthorizeTicket(loginResponse.Token)), 0); + + result = runtime->GrabEdgeEvent(handle); + + UNIT_ASSERT_C(result->Error.empty(), result->Error); + UNIT_ASSERT(result->Token != nullptr); + UNIT_ASSERT_VALUES_EQUAL(result->Token->GetUserSID(), "user1"); + UNIT_ASSERT(result->Token->IsExist("group1")); + UNIT_ASSERT_VALUES_EQUAL(result->Token->GetGroupSIDs().size(), 2); + } + Y_UNIT_TEST(LoginCheckRemovedUser) { using namespace Tests; TPortManager tp; diff --git a/ydb/library/testlib/service_mocks/ldap_mock/ldap_defines.h b/ydb/library/testlib/service_mocks/ldap_mock/ldap_defines.h index 02bc7dd8cb04..e2aa0a1bd755 100644 --- a/ydb/library/testlib/service_mocks/ldap_mock/ldap_defines.h +++ b/ydb/library/testlib/service_mocks/ldap_mock/ldap_defines.h @@ -12,6 +12,7 @@ enum EStatus { SUCCESS = 0x00, PROTOCOL_ERROR = 0x02, INVALID_CREDENTIALS = 0x31, + BUSY = 0x33, }; enum EProtocolOp { From fa7998cc6521bc2bec2768d9f41a9de87b8f76ce Mon Sep 17 00:00:00 2001 From: Maksim Zinal Date: Thu, 7 Nov 2024 17:36:45 +0300 Subject: [PATCH 29/29] experimental patch to stop the influence of lock ranges count over lock escalation --- ydb/core/tx/locks/locks.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ydb/core/tx/locks/locks.cpp b/ydb/core/tx/locks/locks.cpp index 38fc5f1a3115..426d9977844a 100644 --- a/ydb/core/tx/locks/locks.cpp +++ b/ydb/core/tx/locks/locks.cpp @@ -651,7 +651,7 @@ bool TLockLocker::ForceShardLock(const TPathId& tableId) const { auto it = Tables.find(tableId); if (it != Tables.end()) { if (it->second->RangeCount() > LockLimit()) { - return true; + return Locks.size() > LockLimit(); } } return false; @@ -659,8 +659,9 @@ bool TLockLocker::ForceShardLock(const TPathId& tableId) const { bool TLockLocker::ForceShardLock(const TIntrusiveList& readTables) const { for (auto& table : readTables) { - if (table.RangeCount() > LockLimit()) - return true; + if (table.RangeCount() > LockLimit()) { + return Locks.size() > LockLimit(); + } } return false; }