From c09b6b284b90aa80ac209c562802b60918c5f96d Mon Sep 17 00:00:00 2001 From: Grigoriy Pisarenko Date: Tue, 20 Aug 2024 07:03:31 +0000 Subject: [PATCH 1/2] Added ast truncate by 40MB --- .../kqp_finalize_script_actor.cpp | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/ydb/core/kqp/finalize_script_service/kqp_finalize_script_actor.cpp b/ydb/core/kqp/finalize_script_service/kqp_finalize_script_actor.cpp index a32c479ba79b..abf99ba4840e 100644 --- a/ydb/core/kqp/finalize_script_service/kqp_finalize_script_actor.cpp +++ b/ydb/core/kqp/finalize_script_service/kqp_finalize_script_actor.cpp @@ -19,6 +19,8 @@ namespace NKikimr::NKqp { namespace { class TScriptFinalizerActor : public TActorBootstrapped { + static constexpr size_t MAX_ARTIFACTS_SIZE_BYTES = 40_MB; + public: TScriptFinalizerActor(TEvScriptFinalizeRequest::TPtr request, const NKikimrConfig::TQueryServiceConfig& queryServiceConfig, @@ -37,6 +39,14 @@ class TScriptFinalizerActor : public TActorBootstrapped { void CompressScriptArtifacts() const { auto& description = Request->Get()->Description; + + bool truncated = false; + if (size_t planSize = description.QueryPlan.value_or("").size(); description.QueryAst && description.QueryAst->size() + planSize > MAX_ARTIFACTS_SIZE_BYTES) { + size_t toRemove = std::min(description.QueryAst->size() + planSize - MAX_ARTIFACTS_SIZE_BYTES, description.QueryAst->size()); + description.QueryAst = TruncateString(*description.QueryAst, description.QueryAst->size() - toRemove); + truncated = true; + } + auto ast = description.QueryAst; if (Compressor.IsEnabled() && ast) { const auto& [astCompressionMethod, astCompressed] = Compressor.Compress(*ast); @@ -45,12 +55,15 @@ class TScriptFinalizerActor : public TActorBootstrapped { } if (description.QueryAst && description.QueryAst->size() > NDataShard::NLimits::MaxWriteValueSize) { - NYql::TIssue astTruncatedIssue(TStringBuilder() << "Query ast size is " << description.QueryAst->size() << " bytes, that is larger than allowed limit " << NDataShard::NLimits::MaxWriteValueSize << " bytes, ast was truncated"); + description.QueryAst = TruncateString(*ast, NDataShard::NLimits::MaxWriteValueSize - 1_KB); + description.QueryAstCompressionMethod = std::nullopt; + truncated = true; + } + + if (truncated) { + NYql::TIssue astTruncatedIssue("Query ast was truncated"); astTruncatedIssue.SetCode(NYql::DEFAULT_ERROR, NYql::TSeverityIds::S_INFO); description.Issues.AddIssue(astTruncatedIssue); - - description.QueryAst = ast->substr(0, NDataShard::NLimits::MaxWriteValueSize - 1_KB) + "...\n(TRUNCATED)"; - description.QueryAstCompressionMethod = std::nullopt; } } @@ -223,6 +236,11 @@ class TScriptFinalizerActor : public TActorBootstrapped { PassAway(); } +private: + static TString TruncateString(const TString& str, size_t size) { + return str.substr(0, std::min(str.size(), size)) + "...\n(TRUNCATED)"; + } + private: const TActorId ReplyActor; const TString ExecutionId; From 7d8440f7f370c280aaeb76a5e0b17192c761337f Mon Sep 17 00:00:00 2001 From: Grigoriy Pisarenko Date: Tue, 20 Aug 2024 07:18:51 +0000 Subject: [PATCH 2/2] Fixed error message --- .../kqp_finalize_script_actor.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ydb/core/kqp/finalize_script_service/kqp_finalize_script_actor.cpp b/ydb/core/kqp/finalize_script_service/kqp_finalize_script_actor.cpp index abf99ba4840e..0ed438e31ef8 100644 --- a/ydb/core/kqp/finalize_script_service/kqp_finalize_script_actor.cpp +++ b/ydb/core/kqp/finalize_script_service/kqp_finalize_script_actor.cpp @@ -40,11 +40,11 @@ class TScriptFinalizerActor : public TActorBootstrapped { void CompressScriptArtifacts() const { auto& description = Request->Get()->Description; - bool truncated = false; + TString astTruncateDescription; if (size_t planSize = description.QueryPlan.value_or("").size(); description.QueryAst && description.QueryAst->size() + planSize > MAX_ARTIFACTS_SIZE_BYTES) { + astTruncateDescription = TStringBuilder() << "Query artifacts size is " << description.QueryAst->size() + planSize << " bytes (plan + ast), that is larger than allowed limit " << MAX_ARTIFACTS_SIZE_BYTES << " bytes, ast was truncated"; size_t toRemove = std::min(description.QueryAst->size() + planSize - MAX_ARTIFACTS_SIZE_BYTES, description.QueryAst->size()); description.QueryAst = TruncateString(*description.QueryAst, description.QueryAst->size() - toRemove); - truncated = true; } auto ast = description.QueryAst; @@ -55,13 +55,13 @@ class TScriptFinalizerActor : public TActorBootstrapped { } if (description.QueryAst && description.QueryAst->size() > NDataShard::NLimits::MaxWriteValueSize) { + astTruncateDescription = TStringBuilder() << "Query ast size is " << description.QueryAst->size() << " bytes, that is larger than allowed limit " << NDataShard::NLimits::MaxWriteValueSize << " bytes, ast was truncated"; description.QueryAst = TruncateString(*ast, NDataShard::NLimits::MaxWriteValueSize - 1_KB); description.QueryAstCompressionMethod = std::nullopt; - truncated = true; } - if (truncated) { - NYql::TIssue astTruncatedIssue("Query ast was truncated"); + if (astTruncateDescription) { + NYql::TIssue astTruncatedIssue(astTruncateDescription); astTruncatedIssue.SetCode(NYql::DEFAULT_ERROR, NYql::TSeverityIds::S_INFO); description.Issues.AddIssue(astTruncatedIssue); }