diff --git a/ydb/core/fq/libs/control_plane_storage/control_plane_storage.h b/ydb/core/fq/libs/control_plane_storage/control_plane_storage.h index aa0c2f478eda..99fad28211ae 100644 --- a/ydb/core/fq/libs/control_plane_storage/control_plane_storage.h +++ b/ydb/core/fq/libs/control_plane_storage/control_plane_storage.h @@ -40,7 +40,13 @@ namespace NFq { NActors::TActorId ControlPlaneStorageServiceActorId(ui32 nodeId = 0); -NActors::IActor* CreateInMemoryControlPlaneStorageServiceActor(const NConfig::TControlPlaneStorageConfig& config); +NActors::IActor* CreateInMemoryControlPlaneStorageServiceActor( + const NConfig::TControlPlaneStorageConfig& config, + const NYql::TS3GatewayConfig& s3Config, + const NConfig::TCommonConfig& common, + const NConfig::TComputeConfig& computeConfig, + const ::NMonitoring::TDynamicCounterPtr& counters, + const TString& tenantName); NActors::IActor* CreateYdbControlPlaneStorageServiceActor( const NConfig::TControlPlaneStorageConfig& config, diff --git a/ydb/core/fq/libs/control_plane_storage/events/events.h b/ydb/core/fq/libs/control_plane_storage/events/events.h index 2ea1f8911543..bdcdac675d83 100644 --- a/ydb/core/fq/libs/control_plane_storage/events/events.h +++ b/ydb/core/fq/libs/control_plane_storage/events/events.h @@ -392,6 +392,7 @@ struct TEvControlPlaneStorage { // internal messages struct TEvWriteResultDataRequest : NActors::TEventLocal { + using TProto = Fq::Private::WriteTaskResultRequest; TEvWriteResultDataRequest() = default; @@ -411,6 +412,8 @@ struct TEvControlPlaneStorage { struct TEvWriteResultDataResponse : NActors::TEventLocal { static constexpr bool Auditable = false; + using TProto = Fq::Private::WriteTaskResultResult; + explicit TEvWriteResultDataResponse( const Fq::Private::WriteTaskResultResult& record) : Record(record) @@ -434,6 +437,7 @@ struct TEvControlPlaneStorage { }; struct TEvGetTaskRequest : NActors::TEventLocal { + using TProto = Fq::Private::GetTaskRequest; TEvGetTaskRequest() = default; @@ -454,6 +458,8 @@ struct TEvControlPlaneStorage { struct TEvGetTaskResponse : NActors::TEventLocal { static constexpr bool Auditable = false; + using TProto = Fq::Private::GetTaskResult; + explicit TEvGetTaskResponse( const Fq::Private::GetTaskResult& record) : Record(record) @@ -499,6 +505,7 @@ struct TEvControlPlaneStorage { }; struct TEvPingTaskRequest : NActors::TEventLocal { + using TProto = Fq::Private::PingTaskRequest; TEvPingTaskRequest() = default; @@ -519,6 +526,8 @@ struct TEvControlPlaneStorage { struct TEvPingTaskResponse : NActors::TEventLocal { static constexpr bool Auditable = false; + using TProto = Fq::Private::PingTaskResult; + explicit TEvPingTaskResponse( const Fq::Private::PingTaskResult& record) : Record(record) @@ -542,6 +551,7 @@ struct TEvControlPlaneStorage { }; struct TEvNodesHealthCheckRequest : NActors::TEventLocal { + using TProto = Fq::Private::NodesHealthCheckRequest; TEvNodesHealthCheckRequest() = default; @@ -561,6 +571,8 @@ struct TEvControlPlaneStorage { struct TEvNodesHealthCheckResponse : NActors::TEventLocal { static constexpr bool Auditable = false; + using TProto = Fq::Private::NodesHealthCheckResult; + explicit TEvNodesHealthCheckResponse( const Fq::Private::NodesHealthCheckResult& record) : Record(record) diff --git a/ydb/core/fq/libs/control_plane_storage/in_memory_control_plane_storage.cpp b/ydb/core/fq/libs/control_plane_storage/in_memory_control_plane_storage.cpp index f9089b0cac4b..7f715d2ec9f0 100644 --- a/ydb/core/fq/libs/control_plane_storage/in_memory_control_plane_storage.cpp +++ b/ydb/core/fq/libs/control_plane_storage/in_memory_control_plane_storage.cpp @@ -1,59 +1,173 @@ #include "control_plane_storage.h" -#include "util.h" +#include "ydb_control_plane_storage_impl.h" -#include -#include +#include -#include - -#include -#include +#include namespace NFq { -class TInMemoryControlPlaneStorageActor : public NActors::TActor { - struct TKey { +class TInMemoryControlPlaneStorageActor : public NActors::TActor, + public TControlPlaneStorageBase { + struct TScopeKey { TString Scope; TString Id; - bool operator<(const TKey& other) const - { - return tie(Scope, Id) < tie(other.Scope, other.Id); - } + std::strong_ordering operator<=>(const TScopeKey& other) const = default; }; - struct TConfig { - NConfig::TControlPlaneStorageConfig Proto; - TDuration IdempotencyKeyTtl; - TDuration AutomaticQueriesTtl; - TDuration ResultSetsTtl; - TDuration AnalyticsRetryCounterUpdateTime; - TDuration StreamingRetryCounterUpdateTime; - TDuration TaskLeaseTtl; - - TConfig(const NConfig::TControlPlaneStorageConfig& config) - : Proto(FillDefaultParameters(config)) - , IdempotencyKeyTtl(GetDuration(Proto.GetIdempotencyKeysTtl(), TDuration::Minutes(10))) - , AutomaticQueriesTtl(GetDuration(Proto.GetAutomaticQueriesTtl(), TDuration::Days(1))) - , ResultSetsTtl(GetDuration(Proto.GetResultSetsTtl(), TDuration::Days(1))) - , TaskLeaseTtl(GetDuration(Proto.GetTaskLeaseTtl(), TDuration::Seconds(30))) - { - } + struct TQueries { + using TKey = TScopeKey; + + struct TValue { + FederatedQuery::Query Query; + FederatedQuery::Internal::QueryInternal QueryInternal; + TString LastJobId; + TString User; + TString ResultId; + TInstant ResultExpireAt; + ui64 Generation = 0; + TInstant ExpireAt = TInstant::Zero(); + }; + + TMap Values; }; - TConfig Config; - TMap Queries; - TMap Connections; - TMap IdempotencyKeys; // idempotency_key -> created_at + struct TPendingQueries { + struct TKey { + TString Tenant; + TString Scope; + TString QueryId; + + std::strong_ordering operator<=>(const TKey& other) const = default; + }; + + struct TValue { + TRetryLimiter RetryLimiter; + TString Owner; + TInstant AssignedUntil; + TInstant LastSeenAt; + }; + + TMap Values; + }; + + struct TJobs { + struct TKey { + TString Scope; + TString QueryId; + TString JobId; + + std::strong_ordering operator<=>(const TKey& other) const = default; + }; + + struct TValue { + FederatedQuery::Job Job; + TInstant ExpireAt = TInstant::Zero(); + }; + + TMap Values; + }; - static constexpr int64_t InitialRevision = 1; + struct TConnections { + using TKey = TScopeKey; + using TEntity = FederatedQuery::Connection; + + struct TValue { + TEntity Connection; + TString User; + + const TEntity& GetEntity() const { + return Connection; + } + }; + + TMap Values; + }; + + struct TBindings { + using TKey = TScopeKey; + using TEntity = FederatedQuery::Binding; + + struct TValue { + TEntity Binding; + TString User; + + const TEntity& GetEntity() const { + return Binding; + } + }; + + TMap Values; + }; + + struct TIdempotencyKeys { + using TKey = TScopeKey; + + struct TValue { + TString Response; + TInstant ExpireAt = TInstant::Zero(); + }; + + TMap Values; + }; + + struct TResultSets { + struct TKey { + TString ResultId; + i32 ResultSetId; + + std::strong_ordering operator<=>(const TKey& other) const = default; + }; + + struct TValue { + TVector Rows; + TInstant ExpireAt = TInstant::Zero(); + }; + + TMap Values; + }; + + struct TNodes { + static constexpr TDuration TTL = TDuration::Seconds(15); + + struct TKey { + TString Tenant; + ui32 NodeId; + + std::strong_ordering operator<=>(const TKey& other) const = default; + }; + + struct TValue { + Fq::Private::NodeInfo Node; + TInstant ExpireAt = TInstant::Zero(); + }; + + TMap Values; + }; + + using TBase = TControlPlaneStorageBase; + + TQueries Queries; + TPendingQueries PendingQueries; + TJobs Jobs; + TConnections Connections; + TBindings Bindings; + TResultSets ResultSets; + TIdempotencyKeys IdempotencyKeys; + TNodes Nodes; public: - TInMemoryControlPlaneStorageActor(const NConfig::TControlPlaneStorageConfig& config) + TInMemoryControlPlaneStorageActor( + const NConfig::TControlPlaneStorageConfig& config, + const NYql::TS3GatewayConfig& s3Config, + const NConfig::TCommonConfig& common, + const NConfig::TComputeConfig& computeConfig, + const ::NMonitoring::TDynamicCounterPtr& counters, + const TString& tenantName) : TActor(&TThis::StateFunc) - , Config(config) - { - } + , TBase(config, s3Config, common, computeConfig, counters, tenantName) + {} static constexpr char ActorName[] = "YQ_CONTROL_PLANE_STORAGE"; @@ -83,75 +197,258 @@ class TInMemoryControlPlaneStorageActor : public NActors::TActorGet()->Request; - CPS_LOG_D("CreateQueryRequest: " << request.DebugString()); - CleanupIndempotencyKeys(); - auto now = TInstant::Now(); - const TString idempotencyKey = request.idempotency_key(); - if (idempotencyKey && IdempotencyKeys.contains(idempotencyKey)) { - CPS_LOG_D("CreateQueryRequest, idempotency key already exist: " << request.DebugString()); - NYql::TIssue issue = MakeErrorIssue(TIssuesIds::BAD_REQUEST, "idempotency key already exist"); - Send(ev->Sender, new TEvControlPlaneStorage::TEvCreateQueryResponse(NYql::TIssues{issue}), 0, ev->Cookie); - return; - } - - NYql::TIssues issues = ValidateCreateQueryRequest(ev); - if (issues) { - CPS_LOG_D("CreateQueryRequest, validation failed: " << request.DebugString() << " error: " << issues.ToString()); - Send(ev->Sender, new TEvControlPlaneStorage::TEvCreateQueryResponse(issues), 0, ev->Cookie); - return; - } - - const TString user = ev->Get()->User; - const TString scope = ev->Get()->Scope; - const TString queryId = CreateGuidAsString(); - FederatedQuery::Query query; - FederatedQuery::QueryContent& content = *query.mutable_content(); - content = request.content(); - FederatedQuery::QueryMeta& meta = *query.mutable_meta(); - FederatedQuery::CommonMeta& common = *meta.mutable_common(); - common.set_id(queryId); - common.set_created_by(user); - auto timestamp = NProtoInterop::CastToProto(now); - *common.mutable_created_at() = timestamp; - common.set_revision(InitialRevision); - - Queries[{scope, queryId}] = query; - - if (!idempotencyKey) { - IdempotencyKeys[idempotencyKey] = now; - } - - CPS_LOG_D("CreateQueryRequest, success: " << request.DebugString() << " query_id: " << queryId); - FederatedQuery::CreateQueryResult result; - result.set_query_id(queryId); - Send(ev->Sender, new TEvControlPlaneStorage::TEvCreateQueryResponse(result, TAuditDetails{}), 0, ev->Cookie); - } - - void Handle(TEvControlPlaneStorage::TEvListQueriesRequest::TPtr& ev) - { + template + class TCommonRequestContext { + public: + using TResultType = TPrepareResponseResultType; + using TResponse = TResultType::Type; + using TAuditDetails = TResultType::TResponseAuditDetails; + + TCommonRequestContext(TInMemoryControlPlaneStorageActor& self, TEvRequest::TPtr& ev, const TString& cloudId, + const TString& scope, const TString& logPrefix, const TString& requestStr, const TString& responseStr) + : StartTime(TInstant::Now()) + , Event(*ev->Get()) + , Request(Event.Request) + , LogPrefix(logPrefix) + , RequestCounters(self.Counters.GetCounters(cloudId, scope, TYPE_SCOPE, TYPE_COMMON)) + , Self(self) + , EventPtr(ev) + , RequestStr(requestStr) + , ResponseStr(responseStr) + { + Self.Cleanup(); + CPS_LOG_I(RequestStr); + CPS_LOG_T(RequestStr << ":" << LogPrefix); + + RequestCounters.IncInFly(); + RequestCounters.Common->RequestBytes->Add(Event.GetByteSize()); + } + + TCommonRequestContext(TInMemoryControlPlaneStorageActor& self, TEvRequest::TPtr& ev, const TString& requestStr, const TString& responseStr) + : TCommonRequestContext(self, ev, "", "", GetLogPrefix(ev), requestStr, responseStr) + {} + + virtual bool Validate() { + if (const auto& issues = Self.ValidateRequest(EventPtr)) { + Fail("query validation", issues); + return false; + } + return true; + } + + bool IsFailed() const { + return Failed; + } + + void Fail(const TString& logInfo, const NYql::TIssues& issues) const { + Y_ABORT_UNLESS(!Failed, "Can not fail twice"); + CPS_LOG_W(RequestStr << ":" << LogPrefix << logInfo << " FAILED: " << issues.ToOneLineString()); + Self.SendResponseIssues(EventPtr->Sender, issues, EventPtr->Cookie, TInstant::Now() - StartTime, RequestCounters); + } + + virtual ~TCommonRequestContext() { + if (Failed) { + return; + } + + Self.SendResponse( + TStringBuilder() << RequestStr << " - " << ResponseStr, + NActors::TActivationContext::ActorSystem(), + NThreading::MakeFuture(NYdb::TStatus(NYdb::EStatus::SUCCESS, {})), + Self.SelfId(), + EventPtr, + StartTime, + RequestCounters, + [response = Response] { return response; }, + Self.Config->Proto.GetEnableDebugMode() ? std::make_shared() : TDebugInfoPtr{}); + } + + static TString GetLogPrefix(TEvRequest::TPtr& ev) { + return TStringBuilder() << "{" << ev->Get()->Request.DebugString() << "} "; + } + + public: + const TInstant StartTime; + const TEvRequest& Event; + const TEvRequest::TProto Request; + const TString LogPrefix; + TRequestCounters RequestCounters; + TResponse Response; + + protected: + TInMemoryControlPlaneStorageActor& Self; + TEvRequest::TPtr& EventPtr; + const TString RequestStr; + const TString ResponseStr; + bool Failed = false; + }; + + template + class TRequestContext : public TCommonRequestContext { + using TBase = TCommonRequestContext; + + Y_HAS_MEMBER(idempotency_key); + static constexpr bool HasIdempotencyKey = THasidempotency_key::value; + + public: + TRequestContext(TInMemoryControlPlaneStorageActor& self, TEvRequest::TPtr& ev, const TString& requestStr, const TString& responseStr) + : TBase( + self, ev, ev->Get()->CloudId, ev->Get()->Scope, + TStringBuilder() << TBase::GetLogPrefix(ev) << MakeUserInfo(ev->Get()->User, ev->Get()->Token), + requestStr, responseStr + ) + , CloudId(TBase::Event.CloudId) + , Scope(TBase::Event.Scope) + , User(TBase::Event.User) + , Token(TBase::Event.Token) + , Permissions(TBase::Event.Permissions) + , Quotas(TBase::Event.Quotas) + , TenantInfo(TBase::Event.TenantInfo) + , ComputeDatabase(TBase::Event.ComputeDatabase) + { + if constexpr (!std::is_same_v) { + TBase::Response.second.CloudId = CloudId; + } + } + + bool Validate() override { + if (!TBase::Validate()) { + return false; + } + if constexpr (HasIdempotencyKey) { + if (const TString& idempotencyKey = TBase::Request.idempotency_key()) { + if (const auto& value = TBase::Self.GetEntity(TBase::Self.IdempotencyKeys, {Scope, idempotencyKey})) { + if (!TBase::Response.first.ParseFromString(value->Response)) { + TBase::RequestCounters.Common->ParseProtobufError->Inc(); + TBase::Fail("idempotency key parse", {NYql::TIssue("INTERNAL ERROR. Error parsing proto message for idempotency key request. Please contact internal support")}); + } else { + TBase::Response.second.IdempotencyResult = true; + } + return false; + } + } + } + return true; + } + + ~TRequestContext() override { + if (TBase::Failed) { + return; + } + + if constexpr (HasIdempotencyKey) { + if (const TString& idempotencyKey = TBase::Request.idempotency_key()) { + this->Self.AddEntity(this->Self.IdempotencyKeys, {this->Scope, idempotencyKey}, { + .Response = TBase::Response.first.SerializeAsString(), + .ExpireAt = TBase::StartTime + this->Self.Config->IdempotencyKeyTtl + }); + } + } + } + + public: + const TString CloudId; + const TString Scope; + const TString User; + const TString Token; + const TPermissions Permissions; + const TMaybe Quotas; + const TTenantInfo::TPtr TenantInfo; + const FederatedQuery::Internal::ComputeDatabaseInternal ComputeDatabase; + }; + +#define HANDLE_CPS_REQUEST_IMPL(TEvRequest, TEvResponse, TContext, RTS_COUNTERS_ENUM, RTC_COUNTERS_ENUM) \ + using TContext##TEvRequest = TContext< \ + TEvControlPlaneStorage::TEvRequest, TEvControlPlaneStorage::TEvResponse, \ + RTS_COUNTERS_ENUM, RTC_COUNTERS_ENUM>; \ + void Handle(TEvControlPlaneStorage::TEvRequest::TPtr& ev) { \ + TContext##TEvRequest ctx(*this, ev, #TEvRequest, #TEvResponse); \ + if (!ctx.Validate()) { \ + return; \ + } \ + try { \ + Process##TRequest(ctx); \ + } catch (...) { \ + const auto& backtrace = TBackTrace::FromCurrentException().PrintToString(); \ + const auto logError = TStringBuilder() << "pocess "#TEvRequest" call, back trace:\n" << backtrace; \ + ctx.Fail(logError, {NYql::TIssue(CurrentExceptionMessage())}); \ + } \ + } \ + void Process##TRequest(TContext##TEvRequest& ctx) + +#define HANDLE_CPS_REQUEST(TEvRequest, TEvResponse, COUNTERS_ENUM) HANDLE_CPS_REQUEST_IMPL(TEvRequest, TEvResponse, TRequestContext, RTS_##COUNTERS_ENUM, RTC_##COUNTERS_ENUM) + + HANDLE_CPS_REQUEST(TEvCreateQueryRequest, TEvCreateQueryResponse, CREATE_QUERY) { + const auto& [query, job] = GetCreateQueryProtos(ctx.Request, ctx.User, ctx.StartTime); + if (query.ByteSizeLong() > Config->Proto.GetMaxRequestSize()) { + return ctx.Fail("query size validation", {NYql::TIssue(TStringBuilder() << "incoming request exceeded the size limit: " << query.ByteSizeLong() << " of " << Config->Proto.GetMaxRequestSize() << ". Please shorten your request")}); + } + ctx.Response.second.After = query; + + const TString& queryId = query.meta().common().id(); + ctx.Response.first.set_query_id(queryId); + + auto queryInternal = GetQueryInternalProto(ctx.Request, ctx.CloudId, ctx.Token, ctx.Quotas); + const auto queryType = ctx.Request.content().type(); + if (ctx.Request.execute_mode() != FederatedQuery::SAVE) { + *queryInternal.mutable_compute_connection() = ctx.ComputeDatabase.connection(); + FillConnectionsAndBindings( + queryInternal, + queryType, + GetEntities(Connections, ctx.Scope, ctx.User), + GetEntitiesWithVisibilityPriority(Connections, ctx.Scope, ctx.User), + GetEntitiesWithVisibilityPriority(Bindings, ctx.Scope, ctx.User) + ); + } + if (queryInternal.ByteSizeLong() > Config->Proto.GetMaxRequestSize()) { + return ctx.Fail("query internal size validation", {NYql::TIssue(TStringBuilder() << "the size of all connections and bindings in the project exceeded the limit: " << queryInternal.ByteSizeLong() << " of " << Config->Proto.GetMaxRequestSize() << ". Please reduce the number of connections and bindings")}); + } + + const auto& jobId = job.meta().id(); + if (ctx.Request.execute_mode() != FederatedQuery::SAVE) { + AddEntity(Jobs, {ctx.Scope, queryId, jobId}, {job}); + + TRetryLimiter retryLimiter; + retryLimiter.Assign(0, ctx.StartTime, 0.0); + + AddEntity(PendingQueries, { + .Tenant = ctx.TenantInfo->Assign(ctx.CloudId, ctx.Scope, queryType, TenantName), + .Scope = ctx.Scope, + .QueryId = queryId + }, {.RetryLimiter = retryLimiter}); + } + + AddEntity(Queries, {ctx.Scope, queryId}, { + .Query = query, + .QueryInternal = queryInternal, + .LastJobId = jobId, + .User = ctx.User + }); + } + + void Handle(TEvControlPlaneStorage::TEvListQueriesRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyResponse< TEvControlPlaneStorage::TEvListQueriesRequest::TPtr, FederatedQuery::ListQueriesResult, TEvControlPlaneStorage::TEvListQueriesResponse>(ev, "ListQueriesRequest"); } - void Handle(TEvControlPlaneStorage::TEvDescribeQueryRequest::TPtr& ev) - { - SendEmptyResponse< - TEvControlPlaneStorage::TEvDescribeQueryRequest::TPtr, - FederatedQuery::DescribeQueryResult, - TEvControlPlaneStorage::TEvDescribeQueryResponse>(ev, "DescribeQueryRequest"); + HANDLE_CPS_REQUEST(TEvDescribeQueryRequest, TEvDescribeQueryResponse, DESCRIBE_QUERY) { + const auto& query = GetEntity(Queries, {ctx.Scope, ctx.Request.query_id()}); + if (!query) { + return ctx.Fail("find query", {NYql::TIssue("Query does not exist")}); + } + + *ctx.Response.mutable_query() = query->Query; + FillDescribeQueryResult(ctx.Response, query->QueryInternal, ctx.User, ctx.Permissions); } - void Handle(TEvControlPlaneStorage::TEvModifyQueryRequest::TPtr& ev) - { + void Handle(TEvControlPlaneStorage::TEvModifyQueryRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyAuditResponse< TEvControlPlaneStorage::TEvModifyQueryRequest::TPtr, FederatedQuery::ModifyQueryResult, @@ -159,8 +456,8 @@ class TInMemoryControlPlaneStorageActor : public NActors::TActor>(ev, "ModifyQueryRequest"); } - void Handle(TEvControlPlaneStorage::TEvDeleteQueryRequest::TPtr& ev) - { + void Handle(TEvControlPlaneStorage::TEvDeleteQueryRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyAuditResponse< TEvControlPlaneStorage::TEvDeleteQueryRequest::TPtr, FederatedQuery::DeleteQueryResult, @@ -168,8 +465,8 @@ class TInMemoryControlPlaneStorageActor : public NActors::TActor>(ev, "DeleteQueryRequest"); } - void Handle(TEvControlPlaneStorage::TEvControlQueryRequest::TPtr& ev) - { + void Handle(TEvControlPlaneStorage::TEvControlQueryRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyAuditResponse< TEvControlPlaneStorage::TEvControlQueryRequest::TPtr, FederatedQuery::ControlQueryResult, @@ -177,49 +474,98 @@ class TInMemoryControlPlaneStorageActor : public NActors::TActor>(ev, "ControlQueryRequest"); } - void Handle(TEvControlPlaneStorage::TEvGetResultDataRequest::TPtr& ev) - { - SendEmptyResponse< - TEvControlPlaneStorage::TEvGetResultDataRequest::TPtr, - FederatedQuery::GetResultDataResult, - TEvControlPlaneStorage::TEvGetResultDataResponse>(ev, "GetResultDataRequest"); + HANDLE_CPS_REQUEST(TEvGetResultDataRequest, TEvGetResultDataResponse, GET_RESULT_DATA) { + const auto& query = GetEntity(Queries, {ctx.Scope, ctx.Request.query_id()}); + if (!query) { + return ctx.Fail("find query", {NYql::TIssue("Query does not exist")}); + } + + if (!HasViewAccess(GetResultDataReadPerimssions(ctx.Event), query->Query.content().acl().visibility(), query->User, ctx.User)) { + return ctx.Fail("check ACL", {NYql::TIssue("Permission denied")}); + } + + const auto resultSetIndex = ctx.Request.result_set_index(); + const auto& resultSetMeta = query->Query.result_set_meta(); + if (resultSetIndex >= resultSetMeta.size()) { + return ctx.Fail("check result set index", {NYql::TIssue(TStringBuilder() << "Result set index out of bound: " << resultSetIndex << " >= " << resultSetMeta.size())}); + } + + const auto expireAt = query->ResultExpireAt; + if (query->Query.meta().status() != FederatedQuery::QueryMeta::COMPLETED || !expireAt) { + return ctx.Fail("check status", {NYql::TIssue("Result doesn't exist")}); + } + + if (expireAt < TInstant::Now()) { + return ctx.Fail("check expiration", {NYql::TIssue("Result removed by TTL")}); + } + + const auto& result = GetEntity(ResultSets, {query->ResultId, ctx.Request.result_set_index()}); + if (!result) { + return ctx.Fail("get result", {NYql::TIssue("INTERNAL ERROR. Failed to find result set")}); + } + + auto& resultSet = *ctx.Response.mutable_result_set(); + *resultSet.mutable_columns() = resultSetMeta[resultSetIndex].column(); + + const i64 offset = ctx.Request.offset(); + const i64 numberRows = result->Rows.size(); + for (i64 rowId = offset; rowId < offset + ctx.Request.limit() && rowId < numberRows; ++rowId) { + *resultSet.add_rows() = result->Rows[rowId]; + } } - void Handle(TEvControlPlaneStorage::TEvListJobsRequest::TPtr& ev) - { + void Handle(TEvControlPlaneStorage::TEvListJobsRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyResponse< TEvControlPlaneStorage::TEvListJobsRequest::TPtr, FederatedQuery::ListJobsResult, TEvControlPlaneStorage::TEvListJobsResponse>(ev, "ListJobsRequest"); } - void Handle(TEvControlPlaneStorage::TEvCreateConnectionRequest::TPtr& ev) - { - SendEmptyAuditResponse< - TEvControlPlaneStorage::TEvCreateConnectionRequest::TPtr, - FederatedQuery::CreateConnectionResult, - TEvControlPlaneStorage::TEvCreateConnectionResponse, - TAuditDetails>(ev, "CreateConnectionRequest"); + HANDLE_CPS_REQUEST(TEvCreateConnectionRequest, TEvCreateConnectionResponse, CREATE_CONNECTION) { + const auto& content = ctx.Request.content(); + const auto visibility = content.acl().visibility(); + const auto& name = content.name(); + if (!CheckConnectionOrBindingName(Connections, ctx.Scope, ctx.User, visibility, name)) { + return ctx.Fail("check name", {NYql::TIssue("Connection with the same name already exists. Please choose another name")}); + } + if (!CheckConnectionOrBindingName(Bindings, ctx.Scope, ctx.User, visibility, name)) { + return ctx.Fail("check name", {NYql::TIssue("Binding with the same name already exists. Please choose another name")}); + } + if (GetNumberEntitiesByScope(Connections, ctx.Scope) >= Config->Proto.GetMaxCountConnections()) { + return ctx.Fail("check number", {NYql::TIssue(TStringBuilder() << "Too many connections in folder: " << Config->Proto.GetMaxCountConnections() << ". Please remove unused connections")}); + } + + const auto& [connection, _] = GetCreateConnectionProtos(ctx.Request, ctx.CloudId, ctx.User, ctx.StartTime); + ctx.Response.second.After = connection; + + const TString& connectionId = connection.meta().id(); + ctx.Response.first.set_connection_id(connectionId); + + AddEntity(Connections, {ctx.Scope, connectionId}, { + .Connection = connection, + .User = ctx.User + }); } - void Handle(TEvControlPlaneStorage::TEvListConnectionsRequest::TPtr& ev) - { + void Handle(TEvControlPlaneStorage::TEvListConnectionsRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyResponse< TEvControlPlaneStorage::TEvListConnectionsRequest::TPtr, FederatedQuery::ListConnectionsResult, TEvControlPlaneStorage::TEvListConnectionsResponse>(ev, "ListConnectionsRequest"); } - void Handle(TEvControlPlaneStorage::TEvDescribeConnectionRequest::TPtr& ev) - { + void Handle(TEvControlPlaneStorage::TEvDescribeConnectionRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyResponse< TEvControlPlaneStorage::TEvDescribeConnectionRequest::TPtr, FederatedQuery::DescribeConnectionResult, TEvControlPlaneStorage::TEvDescribeConnectionResponse>(ev, "DescribeConnectionRequest"); } - void Handle(TEvControlPlaneStorage::TEvModifyConnectionRequest::TPtr& ev) - { + void Handle(TEvControlPlaneStorage::TEvModifyConnectionRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyAuditResponse< TEvControlPlaneStorage::TEvModifyConnectionRequest::TPtr, FederatedQuery::ModifyConnectionResult, @@ -227,8 +573,8 @@ class TInMemoryControlPlaneStorageActor : public NActors::TActor>(ev, "ModifyConnectionRequest"); } - void Handle(TEvControlPlaneStorage::TEvDeleteConnectionRequest::TPtr& ev) - { + void Handle(TEvControlPlaneStorage::TEvDeleteConnectionRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyAuditResponse< TEvControlPlaneStorage::TEvDeleteConnectionRequest::TPtr, FederatedQuery::DeleteConnectionResult, @@ -236,33 +582,64 @@ class TInMemoryControlPlaneStorageActor : public NActors::TActor>(ev, "DeleteConnectionRequest"); } - void Handle(TEvControlPlaneStorage::TEvCreateBindingRequest::TPtr& ev) - { - SendEmptyAuditResponse< - TEvControlPlaneStorage::TEvCreateBindingRequest::TPtr, - FederatedQuery::CreateBindingResult, - TEvControlPlaneStorage::TEvCreateBindingResponse, - TAuditDetails>(ev, "CreateBindingRequest"); + HANDLE_CPS_REQUEST(TEvCreateBindingRequest, TEvCreateBindingResponse, CREATE_BINDING) { + const auto& content = ctx.Request.content(); + const auto visibility = content.acl().visibility(); + const auto& name = content.name(); + if (!CheckConnectionOrBindingName(Connections, ctx.Scope, ctx.User, visibility, name)) { + return ctx.Fail("check name", {NYql::TIssue("Connection with the same name already exists. Please choose another name")}); + } + if (!CheckConnectionOrBindingName(Bindings, ctx.Scope, ctx.User, visibility, name)) { + return ctx.Fail("check name", {NYql::TIssue("Binding with the same name already exists. Please choose another name")}); + } + if (GetNumberEntitiesByScope(Bindings, ctx.Scope) >= Config->Proto.GetMaxCountBindings()) { + return ctx.Fail("check number", {NYql::TIssue(TStringBuilder() << "Too many bindings in folder: " << Config->Proto.GetMaxCountBindings() << ". Please remove unused bindings")}); + } + + const auto& connection = GetEntity(Connections, {ctx.Scope, content.connection_id()}); + if (!connection) { + return ctx.Fail("check connection", {NYql::TIssue("Connection for binding not found")}); + } + + const auto connectionVisibility = connection->Connection.content().acl().visibility(); + if (content.acl().visibility() == FederatedQuery::Acl::SCOPE && connectionVisibility == FederatedQuery::Acl::PRIVATE) { + return ctx.Fail("check connection ACL", {NYql::TIssue("Binding with SCOPE visibility cannot refer to connection with PRIVATE visibility")}); + } + + if (!HasManageAccess(GetCreateBindingPerimssions(ctx.Event), connectionVisibility, connection->User, ctx.User)) { + return ctx.Fail("check connection ACL", {NYql::TIssue("Permission denied for binding connection")}); + } + + const auto& [binding, _] = GetCreateBindingProtos(ctx.Request, ctx.CloudId, ctx.User, ctx.StartTime); + ctx.Response.second.After = binding; + + const TString& bindingId = binding.meta().id(); + ctx.Response.first.set_binding_id(bindingId); + + AddEntity(Bindings, {ctx.Scope, bindingId}, { + .Binding = binding, + .User = ctx.User + }); } - void Handle(TEvControlPlaneStorage::TEvListBindingsRequest::TPtr& ev) - { + void Handle(TEvControlPlaneStorage::TEvListBindingsRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyResponse< TEvControlPlaneStorage::TEvListBindingsRequest::TPtr, FederatedQuery::ListBindingsResult, TEvControlPlaneStorage::TEvListBindingsResponse>(ev, "ListBindingsRequest"); } - void Handle(TEvControlPlaneStorage::TEvDescribeBindingRequest::TPtr& ev) - { + void Handle(TEvControlPlaneStorage::TEvDescribeBindingRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyResponse< TEvControlPlaneStorage::TEvDescribeBindingRequest::TPtr, FederatedQuery::DescribeBindingResult, TEvControlPlaneStorage::TEvDescribeBindingResponse>(ev, "DescribeBindingRequest"); } - void Handle(TEvControlPlaneStorage::TEvModifyBindingRequest::TPtr& ev) - { + void Handle(TEvControlPlaneStorage::TEvModifyBindingRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyAuditResponse< TEvControlPlaneStorage::TEvModifyBindingRequest::TPtr, FederatedQuery::ModifyBindingResult, @@ -270,8 +647,8 @@ class TInMemoryControlPlaneStorageActor : public NActors::TActor>(ev, "ModifyBindingRequest"); } - void Handle(TEvControlPlaneStorage::TEvDeleteBindingRequest::TPtr& ev) - { + void Handle(TEvControlPlaneStorage::TEvDeleteBindingRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyAuditResponse< TEvControlPlaneStorage::TEvDeleteBindingRequest::TPtr, FederatedQuery::DeleteBindingResult, @@ -279,53 +656,109 @@ class TInMemoryControlPlaneStorageActor : public NActors::TActor>(ev, "DeleteBindingRequest"); } - void Handle(TEvControlPlaneStorage::TEvDescribeJobRequest::TPtr& ev) - { + void Handle(TEvControlPlaneStorage::TEvDescribeJobRequest::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); SendEmptyResponse< TEvControlPlaneStorage::TEvDescribeJobRequest::TPtr, FederatedQuery::DescribeJobResult, TEvControlPlaneStorage::TEvDescribeJobResponse>(ev, "DescribeJobRequest"); } - void Handle(TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr& ev) - { - SendEmptyResponse< - TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr, - NYql::TIssues, - TEvControlPlaneStorage::TEvWriteResultDataResponse>(ev, "WriteResultDataRequest"); + HANDLE_CPS_REQUEST_IMPL(TEvWriteResultDataRequest, TEvWriteResultDataResponse, TCommonRequestContext, RTS_MAX, RTC_WRITE_RESULT_DATA) { + ctx.Response.set_request_id(ctx.Request.request_id()); + + const auto offset = ctx.Request.offset(); + const auto& newRows = ctx.Request.result_set().rows(); + + auto& [resultRows, expireAt] = ResultSets.Values[{ctx.Request.result_id().value(), static_cast(ctx.Request.result_set_id())}]; + expireAt = NProtoInterop::CastFromProto(ctx.Request.deadline()); + + resultRows.resize(std::max(offset + newRows.size(), resultRows.size())); + for (size_t i = offset; const auto& row : newRows) { + resultRows[i++] = row; + } } - void Handle(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev) - { - CPS_LOG_I("GetTaskRequest"); - Fq::Private::GetTaskResult result; - auto event = std::make_unique(result); - NActors::TActivationContext::ActorSystem()->Send(new IEventHandle(ev->Sender, SelfId(), event.release(), 0, ev->Cookie)); + HANDLE_CPS_REQUEST_IMPL(TEvGetTaskRequest, TEvGetTaskResponse, TCommonRequestContext, RTS_MAX, RTC_GET_TASK) { + const auto& tasksInternal = GetActiveTasks(ctx); + + TVector tasks; + tasks.reserve(tasksInternal.size()); + for (const auto& taskInternal : tasksInternal) { + if (const auto& task = AssignTask(ctx, taskInternal)) { + tasks.emplace_back(*task); + } + if (ctx.IsFailed()) { + return; + } + } + + FillGetTaskResult(ctx.Response, tasks); } - void Handle(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev) - { - SendEmptyResponse< - TEvControlPlaneStorage::TEvPingTaskRequest::TPtr, - Fq::Private::PingTaskResult, - TEvControlPlaneStorage::TEvPingTaskResponse>(ev, "PingTaskRequest"); + HANDLE_CPS_REQUEST_IMPL(TEvPingTaskRequest, TEvPingTaskResponse, TCommonRequestContext, RTS_MAX, RTC_PING_TASK) { + const auto& scope = ctx.Request.scope(); + const auto& queryId = ctx.Request.query_id().value(); + + auto query = GetEntity(Queries, {scope, queryId}); + if (!query) { + return ctx.Fail("get query", {NYql::TIssue("INTERNAL ERROR. Query for ping task not found")}); + } + + auto job = GetEntity(Jobs, {scope, queryId, query->LastJobId}); + if (!job) { + return ctx.Fail("get job", {NYql::TIssue("INTERNAL ERROR. Job for ping task not found")}); + } + + auto pendingQuery = GetEntity(PendingQueries, {ctx.Request.tenant(), scope, queryId}); + if (!pendingQuery) { + return ctx.Fail("get pending query", {NYql::TIssue("INTERNAL ERROR. Pending query for ping task not found")}); + } + + auto resuest = ctx.Request; + auto finalStatus = std::make_shared(); + + TDuration backoff = Config->TaskLeaseTtl; + TInstant expireAt = ctx.StartTime + Config->AutomaticQueriesTtl; + UpdateTaskInfo(TActivationContext::ActorSystem(), resuest, finalStatus, query->Query, query->QueryInternal, job->Job, pendingQuery->Owner, pendingQuery->RetryLimiter, backoff, expireAt); + PingTask(ctx, *query, *job, *pendingQuery, backoff, expireAt); + + if (IsTerminalStatus(ctx.Request.status())) { + FillQueryStatistics(finalStatus, query->Query, query->QueryInternal, pendingQuery->RetryLimiter); + } + Send(SelfId(), new TEvControlPlaneStorage::TEvFinalStatusReport( + queryId, finalStatus->JobId, finalStatus->CloudId, scope, std::move(finalStatus->FinalStatistics), + finalStatus->Status, finalStatus->StatusCode, finalStatus->QueryType, finalStatus->Issues, finalStatus->TransientIssues)); } - void Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev) - { - SendEmptyResponse< - TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr, - Fq::Private::NodesHealthCheckResult, - TEvControlPlaneStorage::TEvNodesHealthCheckResponse>(ev, "NodesHealthCheckRequest"); + HANDLE_CPS_REQUEST_IMPL(TEvNodesHealthCheckRequest, TEvNodesHealthCheckResponse, TCommonRequestContext, RTS_MAX, RTC_NODES_HEALTH_CHECK) { + const auto& tenant = ctx.Request.tenant(); + const auto& node = ctx.Request.node(); + + AddEntity(Nodes, {tenant, node.node_id()}, { + .Node = node, + .ExpireAt = ctx.StartTime + TNodes::TTL + }); + + for (const auto& [key, value] : Nodes.Values) { + if (key.Tenant == tenant) { + *ctx.Response.add_nodes() = value.Node; + } + } } +#undef HANDLE_CPS_REQUEST +#undef HANDLE_CPS_REQUEST_IMPL + void Handle(NActors::NMon::TEvHttpInfo::TPtr& ev) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("Unimplemented " << __LINE__); TStringStream str; Send(ev->Sender, new NActors::NMon::TEvHttpInfoRes(str.Str())); } template void SendEmptyResponse(TRequest& ev, std::string logText) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("SendEmptyResponse"); CPS_LOG_I(logText); TResult result = {}; @@ -335,6 +768,7 @@ class TInMemoryControlPlaneStorageActor : public NActors::TActor void SendEmptyAuditResponse(TRequest& ev, std::string logText) { + LOG_YQ_CONTROL_PLANE_STORAGE_CRIT("SendEmptyAuditResponse"); CPS_LOG_I(logText); TResult result = {}; @@ -343,50 +777,250 @@ class TInMemoryControlPlaneStorageActor : public NActors::TActorSend(new IEventHandle(ev->Sender, SelfId(), event.release(), 0, ev->Cookie)); } - NYql::TIssues ValidateCreateQueryRequest(TEvControlPlaneStorage::TEvCreateQueryRequest::TPtr& ev) - { - NYql::TIssues issues; - const FederatedQuery::CreateQueryRequest& request = ev->Get()->Request; - const TString user = ev->Get()->User; - const TString scope = ev->Get()->Scope; - const FederatedQuery::QueryContent& query = request.content(); +private: + template + static std::optional GetEntity(const TTable& table, const TTable::TKey& key) { + const auto it = table.Values.find(key); + if (it == table.Values.end()) { + return std::nullopt; + } + return it->second; + } + + template + static bool AddEntity(TTable& table, const TTable::TKey& key, const TTable::TValue& value) { + return table.Values.emplace(key, value).second; + } + + template + auto GetScopeRange(const TMap& table, const TString& scope) const { + const auto startIt = table.lower_bound({scope, ""}); + + std::ranges::subrange range(startIt, table.end()); + return range | std::views::take_while([scope](auto element) { + return element.first.Scope == scope; + }); + } + + template + auto GetVisibleRange(const TMap& table, const TString& scope, const TString& user, std::optional visibility = std::nullopt) const { + auto range = GetScopeRange(table, scope); + return range | std::views::filter([user, visibility, ignorePrivate = Config->Proto.GetIgnorePrivateSources()](const auto& element) { + const auto entityVisibility = element.second.GetEntity().content().acl().visibility(); + if (ignorePrivate && entityVisibility == FederatedQuery::Acl::PRIVATE) { + return false; + } + if (visibility && entityVisibility != *visibility) { + return false; + } + return entityVisibility == FederatedQuery::Acl::SCOPE || element.second.User == user; + }); + } + + template + TVector GetEntities(const TTable& table, const TString& scope, const TString& user) const { + auto range = GetVisibleRange(table.Values, scope, user) | std::views::transform([](const auto& element) { + return element.second.GetEntity(); + }) | std::views::common; + return {range.begin(), range.end()}; + } + + template + THashMap GetEntitiesWithVisibilityPriority(const TTable& table, const TString& scope, const TString& user) const { + THashMap entities; + for (const auto& [_, value] : GetVisibleRange(table.Values, scope, user)) { + const auto& entity = value.GetEntity(); + const auto visibility = entity.content().acl().visibility(); + const TString& name = entity.content().name(); + if (auto it = entities.find(name); it != entities.end()) { + if (visibility == FederatedQuery::Acl::PRIVATE) { + it->second = entity; + } + } else { + entities[name] = entity; + } + } + return entities; + } + + template + bool CheckConnectionOrBindingName(const TTable& table, const TString& scope, const TString& user, FederatedQuery::Acl::Visibility visibility, const TString& name) const { + auto range = GetVisibleRange(table.Values, scope, user, visibility) | std::views::transform([](const auto& element) { + return element.second.GetEntity().content().name(); + }); + return std::ranges::find(range, name) == range.end(); + } + + template + ui64 GetNumberEntitiesByScope(const TTable& table, const TString& scope) const { + auto range = GetScopeRange(table.Values, scope) | std::views::common; + return std::distance(range.begin(), range.end()); + } + +private: + void Cleanup() { + CleanupTable(Queries); + CleanupTable(Jobs); + CleanupTable(ResultSets); + CleanupTable(IdempotencyKeys); + CleanupTable(Nodes); + } + + template + static void CleanupTable(TTable& table) { + const auto now = TInstant::Now(); + std::erase_if(table.Values, [now](const auto& item) { + const auto expireAt = item.second.ExpireAt; + return expireAt && expireAt < now; + }); + } + +private: + // Get / Ping task utils + + struct TTaskInternal { + TTask Task; + TString Owner; + TRetryLimiter RetryLimiter; + TString TenantName; + bool ShouldAbortTask; + }; + + TVector GetActiveTasks(const TCommonRequestContextTEvGetTaskRequest& ctx) const { + const ui64 tasksBatchSize = Config->Proto.GetTasksBatchSize(); + const TString& tenantName = ctx.Request.tenant(); + + TVector tasks; + tasks.reserve(std::min(tasksBatchSize, PendingQueries.Values.size())); + for (const auto& [key, query] : PendingQueries.Values) { + if (key.Tenant != tenantName || query.AssignedUntil >= ctx.StartTime) { + continue; + } + + TTaskInternal& taskInternal = tasks.emplace_back(); + taskInternal.Owner = ctx.Request.owner_id(); + taskInternal.TenantName = tenantName; + taskInternal.RetryLimiter = query.RetryLimiter; + + auto& task = taskInternal.Task; + task.Scope = key.Scope; + task.QueryId = key.QueryId; + + if (query.Owner) { + CPS_LOG_T("Task (Query): " << task.QueryId << " Lease TIMEOUT, RetryCounterUpdatedAt " << taskInternal.RetryLimiter.RetryCounterUpdatedAt << " LastSeenAt: " << query.LastSeenAt); + taskInternal.ShouldAbortTask = !taskInternal.RetryLimiter.UpdateOnRetry(query.LastSeenAt, Config->TaskLeaseRetryPolicy, ctx.StartTime); + } + task.RetryCount = taskInternal.RetryLimiter.RetryCount; + + CPS_LOG_T("Task (Query): " << task.QueryId << " RetryRate: " << taskInternal.RetryLimiter.RetryRate << " RetryCounter: " << taskInternal.RetryLimiter.RetryCount << " At: " << taskInternal.RetryLimiter.RetryCounterUpdatedAt << (taskInternal.ShouldAbortTask ? " ABORTED" : "")); + + if (tasks.size() >= tasksBatchSize) { + break; + } + } + + std::shuffle(tasks.begin(), tasks.end(), std::default_random_engine(TInstant::Now().MicroSeconds())); + const ui64 numTasksProportion = Config->Proto.GetNumTasksProportion(); + tasks.resize((tasks.size() + numTasksProportion - 1) / numTasksProportion); + + return tasks; + } + + std::optional AssignTask(const TCommonRequestContextTEvGetTaskRequest& ctx, TTaskInternal taskInternal) { + auto& task = taskInternal.Task; - TString error; - if (!request.validate(error)) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, error)); + const auto& query = GetEntity(Queries, {task.Scope, task.QueryId}); + if (!query) { + ctx.Fail("task build", {NYql::TIssue(TStringBuilder() << "INTERNAL ERROR. Not found query for task in scope " << task.Scope << " with query id " << task.QueryId)}); + return std::nullopt; } - if (query.type() == FederatedQuery::QueryContent::QUERY_TYPE_UNSPECIFIED) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "type field is not specified")); + task.Generation = query->Generation + 1; + task.Query = query->Query; + task.Internal = query->QueryInternal; + task.Deadline = TInstant::Now() + (task.Query.content().automatic() ? std::min(Config->AutomaticQueriesTtl, Config->ResultSetsTtl) : Config->ResultSetsTtl); + *task.Internal.mutable_result_ttl() = NProtoInterop::CastToProto(Config->ResultSetsTtl); + + if (Config->Proto.GetDisableCurrentIam()) { + task.Internal.clear_token(); } - if (query.acl().visibility() == FederatedQuery::Acl::VISIBILITY_UNSPECIFIED) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "acl.visibility field is not specified")); + if (taskInternal.ShouldAbortTask) { + AddTransientIssues(task.Query.mutable_transient_issue(), {NYql::TIssue("Query was aborted by system due to high failure rate")}); + task.Query.mutable_meta()->set_status(FederatedQuery::QueryMeta::ABORTING_BY_SYSTEM); } - if (request.ByteSize() > static_cast(Config.Proto.GetMaxRequestSize())) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "Request size exceeded " + ToString(request.ByteSize()) + " out of " + ToString(Config.Proto.GetMaxRequestSize()) + " bytes")); + if (const auto tenantInfo = ctx.Event.TenantInfo) { + const TString& newTenant = tenantInfo->Assign(task.Internal.cloud_id(), task.Scope, task.Query.content().type(), taskInternal.TenantName); + if (newTenant != taskInternal.TenantName) { + UpdateTaskState(ctx, taskInternal, newTenant); + return std::nullopt; + } + if (tenantInfo->TenantState.Value(taskInternal.TenantName, TenantState::Active) != TenantState::Active) { + return std::nullopt; + } } - const uint64_t countQueries = count_if(Queries.begin(), Queries.end(), [scope](const auto& item) { - const auto& [key, value] = item; - return key.Scope == scope; - }); + UpdateTaskState(ctx, taskInternal); + + return task; + } - if (countQueries > Config.Proto.GetMaxCountQueries()) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "The count of the queries exceeds the limit of " + ToString(countQueries) + " out of " + ToString(Config.Proto.GetMaxCountQueries()))); + void UpdateTaskState(const TCommonRequestContextTEvGetTaskRequest& ctx, const TTaskInternal& taskInternal, const TString& newTenant = "") { + const auto& task = taskInternal.Task; + + const auto queryIt = Queries.Values.find({task.Scope, task.QueryId}); + if (queryIt != Queries.Values.end()) { + queryIt->second.Query = task.Query; + queryIt->second.QueryInternal = task.Internal; + queryIt->second.Generation = task.Generation; } - return issues; + const auto pendingIt = PendingQueries.Values.find({taskInternal.TenantName, task.Scope, task.QueryId}); + if (pendingIt != PendingQueries.Values.end()) { + pendingIt->second.Owner = taskInternal.Owner; + pendingIt->second.RetryLimiter = taskInternal.RetryLimiter; + pendingIt->second.AssignedUntil = ctx.StartTime + Config->TaskLeaseTtl; + pendingIt->second.LastSeenAt = ctx.StartTime; + if (newTenant) { + const auto value = pendingIt->second; + PendingQueries.Values.erase(pendingIt); + AddEntity(PendingQueries, {newTenant, task.Scope, task.QueryId}, value); + } + } } - void CleanupIndempotencyKeys() - { - auto now = TInstant::Now(); - erase_if(IdempotencyKeys, [this, now](const auto& item) { - auto const& [idempotencyKey, timestamp] = item; - return timestamp + Config.IdempotencyKeyTtl < now; - }); + void PingTask(const TCommonRequestContextTEvPingTaskRequest& ctx, const TQueries::TValue& query, const TJobs::TValue& job, const TPendingQueries::TValue& pendingQuery, TDuration backoff, TInstant expireAt) { + const auto& scope = ctx.Request.scope(); + const auto& queryId = ctx.Request.query_id().value(); + const auto status = query.Query.meta().status(); + + const auto pendingIt = PendingQueries.Values.find({ctx.Request.tenant(), scope, queryId}); + if (pendingIt != PendingQueries.Values.end()) { + if (IsTerminalStatus(status)) { + PendingQueries.Values.erase(pendingIt); + } else { + pendingIt->second = pendingQuery; + pendingIt->second.AssignedUntil = ctx.StartTime + backoff; + } + } + + const bool automaticFinished = IsTerminalStatus(status) && query.Query.content().automatic(); + const auto jobIt = Jobs.Values.find({scope, queryId, job.Job.meta().id()}); + if (jobIt != Jobs.Values.end()) { + jobIt->second = job; + jobIt->second.ExpireAt = automaticFinished ? expireAt : TInstant::Zero(); + } + + const auto queryIt = Queries.Values.find({scope, queryId}); + if (queryIt != Queries.Values.end()) { + queryIt->second = query; + if (ctx.Request.has_result_id()) { + queryIt->second.ResultId = ctx.Request.result_id().value(); + } + queryIt->second.ResultExpireAt = status == FederatedQuery::QueryMeta::COMPLETED ? NProtoInterop::CastFromProto(ctx.Request.deadline()) : TInstant::Zero(); + queryIt->second.ExpireAt = automaticFinished ? expireAt : TInstant::Zero(); + } } }; @@ -395,8 +1029,14 @@ NActors::TActorId ControlPlaneStorageServiceActorId(ui32 nodeId) { return NActors::TActorId(nodeId, name); } -NActors::IActor* CreateInMemoryControlPlaneStorageServiceActor(const NConfig::TControlPlaneStorageConfig& config) { - return new TInMemoryControlPlaneStorageActor(config); +NActors::IActor* CreateInMemoryControlPlaneStorageServiceActor( + const NConfig::TControlPlaneStorageConfig& config, + const NYql::TS3GatewayConfig& s3Config, + const NConfig::TCommonConfig& common, + const NConfig::TComputeConfig& computeConfig, + const ::NMonitoring::TDynamicCounterPtr& counters, + const TString& tenantName) { + return new TInMemoryControlPlaneStorageActor(config, s3Config, common, computeConfig, counters, tenantName); } } // NFq diff --git a/ydb/core/fq/libs/control_plane_storage/internal/nodes_health_check.cpp b/ydb/core/fq/libs/control_plane_storage/internal/nodes_health_check.cpp index 3b8057b7f8f2..cced0235b898 100644 --- a/ydb/core/fq/libs/control_plane_storage/internal/nodes_health_check.cpp +++ b/ydb/core/fq/libs/control_plane_storage/internal/nodes_health_check.cpp @@ -1,9 +1,17 @@ #include "utils.h" +#include #include namespace NFq { +NYql::TIssues TControlPlaneStorageBase::ValidateRequest(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev) const { + const auto& request = ev->Get()->Request; + const auto& node = request.node(); + + return ValidateNodesHealthCheck(request.tenant(), node.instance_id(), node.hostname()); +} + void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev) { TInstant startTime = TInstant::Now(); @@ -29,8 +37,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealth CPS_LOG_T("NodesHealthCheckRequest: {" << request.DebugString() << "}"); - NYql::TIssues issues = ValidateNodesHealthCheck(tenant, instanceId, hostName); - if (issues) { + if (const auto& issues = ValidateRequest(ev)) { CPS_LOG_W("NodesHealthCheckRequest: {" << request.DebugString() << "} validation FAILED: " << issues.ToOneLineString()); const TDuration delta = TInstant::Now() - startTime; SendResponseIssues(ev->Sender, issues, ev->Cookie, delta, requestCounters); diff --git a/ydb/core/fq/libs/control_plane_storage/internal/task_get.cpp b/ydb/core/fq/libs/control_plane_storage/internal/task_get.cpp index fe8fcab98a6a..e517a51a5824 100644 --- a/ydb/core/fq/libs/control_plane_storage/internal/task_get.cpp +++ b/ydb/core/fq/libs/control_plane_storage/internal/task_get.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -247,6 +248,85 @@ TDuration ExtractLimit(const TTask& task) { return executionLimit; } +NYql::TIssues TControlPlaneStorageBase::ValidateRequest(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev) const +{ + const auto& request = ev->Get()->Request; + NYql::TIssues issues = ValidateGetTask(request.owner_id(), request.host()); + + if (!ev->Get()->TenantInfo) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::NOT_READY, "Control Plane is not ready yet. Please retry later.")); + } + + return issues; +} + +void TControlPlaneStorageBase::FillGetTaskResult(Fq::Private::GetTaskResult& result, const TVector& tasks) const +{ + for (const auto& task : tasks) { + const auto& queryType = task.Query.content().type(); + if (queryType != FederatedQuery::QueryContent::ANALYTICS && queryType != FederatedQuery::QueryContent::STREAMING) { //TODO: fix + ythrow yexception() + << "query type " + << FederatedQuery::QueryContent::QueryType_Name(queryType) + << " unsupported"; + } + + auto* newTask = result.add_tasks(); + newTask->set_query_type(queryType); + newTask->set_query_syntax(task.Query.content().syntax()); + newTask->set_execute_mode(task.Query.meta().execute_mode()); + newTask->set_state_load_mode(task.Internal.state_load_mode()); + auto* queryId = newTask->mutable_query_id(); + queryId->set_value(task.Query.meta().common().id()); + newTask->set_streaming(queryType == FederatedQuery::QueryContent::STREAMING); + newTask->set_text(task.Query.content().text()); + *newTask->mutable_connection() = task.Internal.connection(); + *newTask->mutable_binding() = task.Internal.binding(); + newTask->set_user_token(task.Internal.token()); + newTask->set_user_id(task.Query.meta().common().created_by()); + newTask->set_generation(task.Generation); + newTask->set_status(task.Query.meta().status()); + *newTask->mutable_created_topic_consumers() = task.Internal.created_topic_consumers(); + newTask->mutable_sensor_labels()->insert({"cloud_id", task.Internal.cloud_id()}); + newTask->mutable_sensor_labels()->insert({"scope", task.Scope}); + newTask->set_automatic(task.Query.content().automatic()); + newTask->set_query_name(task.Query.content().name()); + *newTask->mutable_deadline() = NProtoInterop::CastToProto(task.Deadline); + newTask->mutable_disposition()->CopyFrom(task.Internal.disposition()); + newTask->set_result_limit(task.Internal.result_limit()); + *newTask->mutable_execution_limit() = NProtoInterop::CastToProto(ExtractLimit(task)); + *newTask->mutable_request_started_at() = task.Query.meta().started_at(); + *newTask->mutable_request_submitted_at() = task.Query.meta().submitted_at(); + + newTask->set_restart_count(task.RetryCount); + auto* jobId = newTask->mutable_job_id(); + jobId->set_value(task.Query.meta().last_job_id()); + + for (const auto& connection: task.Internal.connection()) { + const auto serviceAccountId = ExtractServiceAccountId(connection); + if (!serviceAccountId) { + continue; + } + auto* account = newTask->add_service_accounts(); + account->set_value(serviceAccountId); + } + + *newTask->mutable_dq_graph() = task.Internal.dq_graph(); + newTask->set_dq_graph_index(task.Internal.dq_graph_index()); + *newTask->mutable_dq_graph_compressed() = task.Internal.dq_graph_compressed(); + + *newTask->mutable_result_set_meta() = task.Query.result_set_meta(); + newTask->set_scope(task.Scope); + *newTask->mutable_resources() = task.Internal.resources(); + + newTask->set_execution_id(task.Internal.execution_id()); + newTask->set_operation_id(task.Internal.operation_id()); + *newTask->mutable_compute_connection() = task.Internal.compute_connection(); + *newTask->mutable_result_ttl() = task.Internal.result_ttl(); + *newTask->mutable_parameters() = task.Query.content().parameters(); + } +} + void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev) { TInstant startTime = TInstant::Now(); @@ -262,13 +342,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetTaskRequ CPS_LOG_T("GetTaskRequest: {" << request.DebugString() << "}"); - NYql::TIssues issues = ValidateGetTask(owner, hostName); - - if (!ev->Get()->TenantInfo) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::NOT_READY, "Control Plane is not ready yet. Please retry later.")); - } - - if (issues) { + if (const auto& issues = ValidateRequest(ev)) { CPS_LOG_W("GetTaskRequest: {" << request.DebugString() << "} FAILED: " << issues.ToOneLineString()); const TDuration delta = TInstant::Now() - startTime; SendResponseIssues(ev->Sender, issues, ev->Cookie, delta, requestCounters); @@ -364,15 +438,17 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetTaskRequ responseTasks=responseTasks] (const auto& readFuture) mutable { try { - if (!readFuture.GetValue().IsSuccess()) + if (!readFuture.GetValue().IsSuccess()) { return readFuture; + } } catch (...) { return readFuture; } auto pickTaskParams = prepareParams(*resultSets); - if (pickTaskParams.empty()) + if (pickTaskParams.empty()) { return readFuture; + } auto debugInfos = std::make_shared>(pickTaskParams.size()); if (Config->Proto.GetEnableDebugMode()) { @@ -410,74 +486,9 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetTaskRequ }); }); - auto prepare = [response] { + auto prepare = [this, response] { Fq::Private::GetTaskResult result; - const auto& tasks = std::get<0>(*response); - - for (const auto& task : tasks) { - const auto& queryType = task.Query.content().type(); - if (queryType != FederatedQuery::QueryContent::ANALYTICS && queryType != FederatedQuery::QueryContent::STREAMING) { //TODO: fix - ythrow yexception() - << "query type " - << FederatedQuery::QueryContent::QueryType_Name(queryType) - << " unsupported"; - } - - auto* newTask = result.add_tasks(); - newTask->set_query_type(queryType); - newTask->set_query_syntax(task.Query.content().syntax()); - newTask->set_execute_mode(task.Query.meta().execute_mode()); - newTask->set_state_load_mode(task.Internal.state_load_mode()); - auto* queryId = newTask->mutable_query_id(); - queryId->set_value(task.Query.meta().common().id()); - newTask->set_streaming(queryType == FederatedQuery::QueryContent::STREAMING); - newTask->set_text(task.Query.content().text()); - *newTask->mutable_connection() = task.Internal.connection(); - *newTask->mutable_binding() = task.Internal.binding(); - newTask->set_user_token(task.Internal.token()); - newTask->set_user_id(task.Query.meta().common().created_by()); - newTask->set_generation(task.Generation); - newTask->set_status(task.Query.meta().status()); - *newTask->mutable_created_topic_consumers() = task.Internal.created_topic_consumers(); - newTask->mutable_sensor_labels()->insert({"cloud_id", task.Internal.cloud_id()}); - newTask->mutable_sensor_labels()->insert({"scope", task.Scope}); - newTask->set_automatic(task.Query.content().automatic()); - newTask->set_query_name(task.Query.content().name()); - *newTask->mutable_deadline() = NProtoInterop::CastToProto(task.Deadline); - newTask->mutable_disposition()->CopyFrom(task.Internal.disposition()); - newTask->set_result_limit(task.Internal.result_limit()); - *newTask->mutable_execution_limit() = NProtoInterop::CastToProto(ExtractLimit(task)); - *newTask->mutable_request_started_at() = task.Query.meta().started_at(); - *newTask->mutable_request_submitted_at() = task.Query.meta().submitted_at(); - - newTask->set_restart_count(task.RetryCount); - auto* jobId = newTask->mutable_job_id(); - jobId->set_value(task.Query.meta().last_job_id()); - - for (const auto& connection: task.Internal.connection()) { - const auto serviceAccountId = ExtractServiceAccountId(connection); - if (!serviceAccountId) { - continue; - } - auto* account = newTask->add_service_accounts(); - account->set_value(serviceAccountId); - } - - *newTask->mutable_dq_graph() = task.Internal.dq_graph(); - newTask->set_dq_graph_index(task.Internal.dq_graph_index()); - *newTask->mutable_dq_graph_compressed() = task.Internal.dq_graph_compressed(); - - *newTask->mutable_result_set_meta() = task.Query.result_set_meta(); - newTask->set_scope(task.Scope); - *newTask->mutable_resources() = task.Internal.resources(); - - newTask->set_execution_id(task.Internal.execution_id()); - newTask->set_operation_id(task.Internal.operation_id()); - *newTask->mutable_compute_connection() = task.Internal.compute_connection(); - *newTask->mutable_result_ttl() = task.Internal.result_ttl(); - *newTask->mutable_parameters() = task.Query.content().parameters(); - } - + FillGetTaskResult(result, std::get<0>(*response)); return result; }; auto success = SendResponse diff --git a/ydb/core/fq/libs/control_plane_storage/internal/task_ping.cpp b/ydb/core/fq/libs/control_plane_storage/internal/task_ping.cpp index 6dc0e76d93c7..0ea26139ce77 100644 --- a/ydb/core/fq/libs/control_plane_storage/internal/task_ping.cpp +++ b/ydb/core/fq/libs/control_plane_storage/internal/task_ping.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -45,36 +46,15 @@ THashMap DeserializeFlatStats(const google::protobuf::RepeatedPtrF } -struct TPingTaskParams { - TString Query; - TParams Params; - const std::function(const std::vector&)> Prepare; - std::shared_ptr> MeteringRecords; -}; - -struct TFinalStatus { - FederatedQuery::QueryMeta::ComputeStatus Status = FederatedQuery::QueryMeta::COMPUTE_STATUS_UNSPECIFIED; - NYql::NDqProto::StatusIds::StatusCode StatusCode = NYql::NDqProto::StatusIds::UNSPECIFIED; - FederatedQuery::QueryContent::QueryType QueryType = FederatedQuery::QueryContent::QUERY_TYPE_UNSPECIFIED; - NYql::TIssues Issues; - NYql::TIssues TransientIssues; - StatsValuesList FinalStatistics; - TString CloudId; - TString JobId; -}; - -TPingTaskParams ConstructHardPingTask( +TYdbControlPlaneStorageActor::TPingTaskParams TYdbControlPlaneStorageActor::ConstructHardPingTask( const Fq::Private::PingTaskRequest& request, std::shared_ptr response, - const TString& tablePathPrefix, const TDuration& automaticQueriesTtl, const TDuration& taskLeaseTtl, - const THashMap& retryPolicies, ::NMonitoring::TDynamicCounterPtr rootCounters, - uint64_t maxRequestSize, bool dumpRawStatistics, const std::shared_ptr& finalStatus, - const TRequestCommonCountersPtr& commonCounters) { + const std::shared_ptr& finalStatus, const TRequestCommonCountersPtr& commonCounters) const { auto scope = request.scope(); auto query_id = request.query_id().value(); - auto counters = rootCounters->GetSubgroup("scope", scope)->GetSubgroup("query_id", query_id); + auto counters = Counters.Counters->GetSubgroup("scope", scope)->GetSubgroup("query_id", query_id); - TSqlQueryBuilder readQueryBuilder(tablePathPrefix, "HardPingTask(read)"); + TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "HardPingTask(read)"); readQueryBuilder.AddString("tenant", request.tenant()); readQueryBuilder.AddString("scope", scope); readQueryBuilder.AddString("query_id", query_id); @@ -146,297 +126,12 @@ TPingTaskParams ConstructHardPingTask( ); } - TMaybe queryStatus; - if (request.status() != FederatedQuery::QueryMeta::COMPUTE_STATUS_UNSPECIFIED) { - queryStatus = request.status(); - } - TMaybe issues; - if (request.issues().size() > 0) { - NYql::TIssues requestIssues; - NYql::IssuesFromMessage(request.issues(), requestIssues); - issues = requestIssues; - } - TMaybe transientIssues; - if (request.transient_issues().size() > 0) { - NYql::TIssues requestTransientIssues; - NYql::IssuesFromMessage(request.transient_issues(), requestTransientIssues); - transientIssues = requestTransientIssues; - } // running query us locked for lease period - TDuration backoff = taskLeaseTtl; - - if (request.resign_query()) { - if (request.status_code() == NYql::NDqProto::StatusIds::UNSPECIFIED && internal.pending_status_code() != NYql::NDqProto::StatusIds::UNSPECIFIED) { - request.set_status_code(internal.pending_status_code()); - internal.clear_pending_status_code(); - internal.clear_execution_id(); - internal.clear_operation_id(); - } - - TRetryPolicyItem policy(0, 0, TDuration::Seconds(1), TDuration::Zero()); - auto it = retryPolicies.find(request.status_code()); - auto policyFound = it != retryPolicies.end(); - if (policyFound) { - policy = it->second; - } - - auto now = TInstant::Now(); - auto executionDeadline = TInstant::Max(); - - auto submittedAt = NProtoInterop::CastFromProto(query.meta().submitted_at()); - auto executionTtl = NProtoInterop::CastFromProto(internal.execution_ttl()); - if (submittedAt && executionTtl) { - executionDeadline = submittedAt + executionTtl; - } - - if (retryLimiter.UpdateOnRetry(now, policy) && now < executionDeadline) { - queryStatus.Clear(); - // failing query is throttled for backoff period - backoff = policy.BackoffPeriod * (retryLimiter.RetryRate + 1); - owner = ""; - if (!transientIssues) { - transientIssues.ConstructInPlace(); - } - TStringBuilder builder; - builder << "Query failed with code " << NYql::NDqProto::StatusIds_StatusCode_Name(request.status_code()) - << " and will be restarted (RetryCount: " << retryLimiter.RetryCount << ")" - << " at " << now; - transientIssues->AddIssue(NYql::TIssue(builder)); - } else { - // failure query should be processed instantly - queryStatus = FederatedQuery::QueryMeta::FAILING; - backoff = TDuration::Zero(); - TStringBuilder builder; - builder << "Query failed with code " << NYql::NDqProto::StatusIds_StatusCode_Name(request.status_code()); - if (policy.RetryCount) { - builder << " (" << retryLimiter.LastError << ")"; - } - builder << " at " << now; - - // in case of problems with finalization, do not change the issues - if (query.meta().status() == FederatedQuery::QueryMeta::FAILING || query.meta().status() == FederatedQuery::QueryMeta::ABORTING_BY_SYSTEM || query.meta().status() == FederatedQuery::QueryMeta::ABORTING_BY_USER) { - if (issues) { - transientIssues->AddIssues(*issues); - } - transientIssues->AddIssue(NYql::TIssue(builder)); - } else { - if (!issues) { - issues.ConstructInPlace(); - } - auto issue = NYql::TIssue(builder); - if (query.issue().size() > 0 && request.issues().empty()) { - NYql::TIssues queryIssues; - NYql::IssuesFromMessage(query.issue(), queryIssues); - for (auto& subIssue : queryIssues) { - issue.AddSubIssue(MakeIntrusive(subIssue)); - } - } - if (transientIssues) { - for (auto& subIssue : *transientIssues) { - issue.AddSubIssue(MakeIntrusive(subIssue)); - } - transientIssues.Clear(); - } - issues->AddIssue(issue); - } - } - CPS_LOG_AS_D(*actorSystem, "PingTaskRequest (resign): " << (!policyFound ? " DEFAULT POLICY" : "") << (owner ? " FAILURE " : " ") << NYql::NDqProto::StatusIds_StatusCode_Name(request.status_code()) << " " << retryLimiter.RetryCount << " " << retryLimiter.RetryCounterUpdatedAt << " " << backoff); - } - - if (queryStatus) { - query.mutable_meta()->set_status(*queryStatus); - job.mutable_query_meta()->set_status(*queryStatus); - } - - if (request.status_code() != NYql::NDqProto::StatusIds::UNSPECIFIED) { - internal.set_status_code(request.status_code()); - } - - if (request.pending_status_code() != NYql::NDqProto::StatusIds::UNSPECIFIED) { - internal.set_pending_status_code(request.pending_status_code()); - } - - if (issues) { - NYql::IssuesToMessage(*issues, query.mutable_issue()); - NYql::IssuesToMessage(*issues, job.mutable_issue()); - } - - if (transientIssues) { - AddTransientIssues(query.mutable_transient_issue(), std::move(*transientIssues)); - } - - if (request.internal_issues().size()) { - *internal.mutable_internal_issue() = request.internal_issues(); - } - - if (request.statistics()) { - TString statistics = request.statistics(); - if (request.flat_stats_size() == 0) { - internal.clear_statistics(); - // TODO: remove once V1 and V2 stats go the same way - PackStatisticsToProtobuf(*internal.mutable_statistics(), statistics, TInstant::Now() - NProtoInterop::CastFromProto(job.meta().created_at())); - } - - // global dumpRawStatistics will be removed with YQv1 - if (!dumpRawStatistics && !request.dump_raw_statistics()) { - try { - statistics = GetPrettyStatistics(statistics); - } catch (const std::exception&) { - // LOG_AS? - CPS_LOG_E("Error on statistics prettification: " << CurrentExceptionMessage()); - } - } - *query.mutable_statistics()->mutable_json() = statistics; - *job.mutable_statistics()->mutable_json() = statistics; - } - - if (request.current_load()) { - internal.set_current_load(request.current_load()); - } - - if (request.timeline()) { - internal.set_timeline(request.timeline()); - } - - if (request.flat_stats_size() != 0) { - internal.clear_statistics(); - auto stats = DeserializeFlatStats(request.flat_stats()); - PackStatisticsToProtobuf(*internal.mutable_statistics(), stats, TInstant::Now() - NProtoInterop::CastFromProto(job.meta().created_at())); - } - - if (!request.result_set_meta().empty()) { - // we will overwrite result_set_meta's COMPLETELY - *query.mutable_result_set_meta() = request.result_set_meta(); - *job.mutable_result_set_meta() = request.result_set_meta(); - } - - if (request.ast()) { - query.mutable_ast()->set_data(request.ast()); - job.mutable_ast()->set_data(request.ast()); - } - - if (request.plan()) { - query.mutable_plan()->set_json(request.plan()); - job.mutable_plan()->set_json(request.plan()); - } - - if (request.ast_compressed().data()) { - internal.mutable_ast_compressed()->set_method(request.ast_compressed().method()); - internal.mutable_ast_compressed()->set_data(request.ast_compressed().data()); - // todo: keep AST compressed in JobInternal - // job.mutable_ast()->set_data(request.ast()); - } - - if (request.plan_compressed().data()) { - internal.mutable_plan_compressed()->set_method(request.plan_compressed().method()); - internal.mutable_plan_compressed()->set_data(request.plan_compressed().data()); - // todo: keep plan compressed in JobInternal - // job.mutable_plan()->set_json(request.plan()); - } - - if (request.has_started_at()) { - *query.mutable_meta()->mutable_started_at() = request.started_at(); - *job.mutable_query_meta()->mutable_started_at() = request.started_at(); - } - - if (request.has_finished_at()) { - *query.mutable_meta()->mutable_finished_at() = request.finished_at(); - *job.mutable_query_meta()->mutable_finished_at() = request.finished_at(); - if (!query.meta().has_started_at()) { - *query.mutable_meta()->mutable_started_at() = request.finished_at(); - *job.mutable_query_meta()->mutable_started_at() = request.finished_at(); - } - } - - TInstant expireAt = TInstant::Now() + automaticQueriesTtl; - if (IsTerminalStatus(query.meta().status()) && query.content().automatic()) { - *query.mutable_meta()->mutable_expire_at() = NProtoInterop::CastToProto(expireAt); - *job.mutable_query_meta()->mutable_expire_at() = NProtoInterop::CastToProto(expireAt); - *job.mutable_expire_at() = NProtoInterop::CastToProto(expireAt); - } - - if (query.meta().status() == FederatedQuery::QueryMeta::COMPLETED) { - *query.mutable_meta()->mutable_result_expire_at() = request.deadline(); - } - - if (request.state_load_mode()) { - internal.set_state_load_mode(request.state_load_mode()); - if (request.state_load_mode() == FederatedQuery::FROM_LAST_CHECKPOINT) { // Saved checkpoint - query.mutable_meta()->set_has_saved_checkpoints(true); - } - } - - if (request.has_disposition()) { - *internal.mutable_disposition() = request.disposition(); - } - - if (request.status() && IsTerminalStatus(request.status())) { - internal.clear_created_topic_consumers(); - // internal.clear_dq_graph(); keep for debug - internal.clear_dq_graph_index(); - // internal.clear_execution_id(); keep for debug - // internal.clear_operation_id(); keep for debug - } - - if (!request.created_topic_consumers().empty()) { - std::set mergedConsumers; - for (auto&& c : *internal.mutable_created_topic_consumers()) { - mergedConsumers.emplace(std::move(c)); - } - for (const auto& c : request.created_topic_consumers()) { - mergedConsumers.emplace(c); - } - internal.clear_created_topic_consumers(); - for (auto&& c : mergedConsumers) { - *internal.add_created_topic_consumers() = std::move(c); - } - } - - if (!request.execution_id().empty()) { - internal.set_execution_id(request.execution_id()); - } - - if (!request.operation_id().empty()) { - internal.set_operation_id(request.operation_id()); - } + TDuration backoff = Config->TaskLeaseTtl; + TInstant expireAt = TInstant::Now() + Config->AutomaticQueriesTtl; + UpdateTaskInfo(actorSystem, request, finalStatus, query, internal, job, owner, retryLimiter, backoff, expireAt); - if (!request.dq_graph().empty()) { - *internal.mutable_dq_graph() = request.dq_graph(); - } - - if (!request.dq_graph_compressed().empty()) { - *internal.mutable_dq_graph_compressed() = request.dq_graph_compressed(); - } - - if (request.dq_graph_index()) { - internal.set_dq_graph_index(request.dq_graph_index()); - } - - if (request.has_resources()) { - *internal.mutable_resources() = request.resources(); - } - - if (job.ByteSizeLong() > maxRequestSize) { - ythrow NYql::TCodeLineException(TIssuesIds::BAD_REQUEST) << "Job proto exceeded the size limit: " << job.ByteSizeLong() << " of " << maxRequestSize << " " << TSizeFormatPrinter(job).ToString(); - } - - if (query.ByteSizeLong() > maxRequestSize) { - ythrow NYql::TCodeLineException(TIssuesIds::BAD_REQUEST) << "Query proto exceeded the size limit: " << query.ByteSizeLong() << " of " << maxRequestSize << " " << TSizeFormatPrinter(query).ToString(); - } - - if (internal.ByteSizeLong() > maxRequestSize) { - ythrow NYql::TCodeLineException(TIssuesIds::BAD_REQUEST) << "QueryInternal proto exceeded the size limit: " << internal.ByteSizeLong() << " of " << maxRequestSize << " " << TSizeFormatPrinter(internal).ToString(); - } - - finalStatus->Status = query.meta().status(); - finalStatus->QueryType = query.content().type(); - finalStatus->StatusCode = internal.status_code(); - finalStatus->CloudId = internal.cloud_id(); - finalStatus->JobId = jobId; - NYql::IssuesFromMessage(query.issue(), finalStatus->Issues); - NYql::IssuesFromMessage(query.transient_issue(), finalStatus->TransientIssues); - - TSqlQueryBuilder writeQueryBuilder(tablePathPrefix, "HardPingTask(write)"); + TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "HardPingTask(write)"); writeQueryBuilder.AddString("tenant", request.tenant()); writeQueryBuilder.AddString("scope", request.scope()); writeQueryBuilder.AddString("job_id", jobId); @@ -531,18 +226,7 @@ TPingTaskParams ConstructHardPingTask( // YQv2 may not provide statistics with terminal status, use saved one statistics = query.statistics().json(); } - finalStatus->FinalStatistics = ExtractStatisticsFromProtobuf(internal.statistics()); - finalStatus->FinalStatistics.push_back(std::make_pair("IsAutomatic", query.content().automatic())); - if (query.content().name().Contains("DataLens YQ query")) { - finalStatus->FinalStatistics.push_back(std::make_pair("IsDataLens", 1)); - } else if (query.content().name().Contains("Audit-trails")) { - finalStatus->FinalStatistics.push_back(std::make_pair("IsAuditTrails", 1)); - } else if (query.content().name().Contains("Query from YDB SDK")) { - finalStatus->FinalStatistics.push_back(std::make_pair("IsSDK", 1)); - } - finalStatus->FinalStatistics.push_back(std::make_pair("RetryCount", retryLimiter.RetryCount)); - finalStatus->FinalStatistics.push_back(std::make_pair("RetryRate", retryLimiter.RetryRate * 100)); - finalStatus->FinalStatistics.push_back(std::make_pair("Load", internal.current_load())); + FillQueryStatistics(finalStatus, query, internal, retryLimiter); auto records = GetMeteringRecords(statistics, isBillable, jobId, request.scope(), HostName()); meteringRecords->swap(records); @@ -558,10 +242,10 @@ TPingTaskParams ConstructHardPingTask( return {readQuery.Sql, readQuery.Params, prepareParams, meteringRecords}; } -TPingTaskParams ConstructSoftPingTask( +TYdbControlPlaneStorageActor::TPingTaskParams TYdbControlPlaneStorageActor::ConstructSoftPingTask( const Fq::Private::PingTaskRequest& request, std::shared_ptr response, - const TString& tablePathPrefix, const TDuration& taskLeaseTtl, const TRequestCommonCountersPtr& commonCounters) { - TSqlQueryBuilder readQueryBuilder(tablePathPrefix, "SoftPingTask(read)"); + const TRequestCommonCountersPtr& commonCounters) const { + TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "SoftPingTask(read)"); readQueryBuilder.AddString("tenant", request.tenant()); readQueryBuilder.AddString("scope", request.scope()); readQueryBuilder.AddString("query_id", request.query_id().value()); @@ -603,11 +287,11 @@ TPingTaskParams ConstructSoftPingTask( } } - TInstant ttl = TInstant::Now() + taskLeaseTtl; + TInstant ttl = TInstant::Now() + Config->TaskLeaseTtl; response->set_action(internal.action()); *response->mutable_expired_at() = google::protobuf::util::TimeUtil::MillisecondsToTimestamp(ttl.MilliSeconds()); - TSqlQueryBuilder writeQueryBuilder(tablePathPrefix, "SoftPingTask(write)"); + TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "SoftPingTask(write)"); writeQueryBuilder.AddTimestamp("now", TInstant::Now()); writeQueryBuilder.AddTimestamp("ttl", ttl); writeQueryBuilder.AddString("tenant", request.tenant()); @@ -626,6 +310,330 @@ TPingTaskParams ConstructSoftPingTask( return {readQuery.Sql, readQuery.Params, prepareParams, std::shared_ptr>{}}; } +NYql::TIssues TControlPlaneStorageBase::ValidateRequest(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev) const { + const Fq::Private::PingTaskRequest& request = ev->Get()->Request; + + NYql::TIssues issues = ValidatePingTask(request.scope(), request.query_id().value(), request.owner_id(), NProtoInterop::CastFromProto(request.deadline()), Config->ResultSetsTtl); + + const auto tenantInfo = ev->Get()->TenantInfo; + if (tenantInfo && tenantInfo->TenantState.Value(request.tenant(), TenantState::Active) == TenantState::Idle) { + issues.AddIssue("Tenant is idle, no processing is allowed"); + } + + return issues; +} + +void TControlPlaneStorageBase::UpdateTaskInfo( + NActors::TActorSystem* actorSystem, Fq::Private::PingTaskRequest& request, const std::shared_ptr& finalStatus, FederatedQuery::Query& query, + FederatedQuery::Internal::QueryInternal& internal, FederatedQuery::Job& job, TString& owner, + TRetryLimiter& retryLimiter, TDuration& backoff, TInstant& expireAt) const +{ + TMaybe queryStatus; + if (request.status() != FederatedQuery::QueryMeta::COMPUTE_STATUS_UNSPECIFIED) { + queryStatus = request.status(); + } + TMaybe issues; + if (request.issues().size() > 0) { + NYql::TIssues requestIssues; + NYql::IssuesFromMessage(request.issues(), requestIssues); + issues = requestIssues; + } + TMaybe transientIssues; + if (request.transient_issues().size() > 0) { + NYql::TIssues requestTransientIssues; + NYql::IssuesFromMessage(request.transient_issues(), requestTransientIssues); + transientIssues = requestTransientIssues; + } + + if (request.resign_query()) { + if (request.status_code() == NYql::NDqProto::StatusIds::UNSPECIFIED && internal.pending_status_code() != NYql::NDqProto::StatusIds::UNSPECIFIED) { + request.set_status_code(internal.pending_status_code()); + internal.clear_pending_status_code(); + internal.clear_execution_id(); + internal.clear_operation_id(); + } + + TRetryPolicyItem policy(0, 0, TDuration::Seconds(1), TDuration::Zero()); + auto it = Config->RetryPolicies.find(request.status_code()); + auto policyFound = it != Config->RetryPolicies.end(); + if (policyFound) { + policy = it->second; + } + + auto now = TInstant::Now(); + auto executionDeadline = TInstant::Max(); + + auto submittedAt = NProtoInterop::CastFromProto(query.meta().submitted_at()); + auto executionTtl = NProtoInterop::CastFromProto(internal.execution_ttl()); + if (submittedAt && executionTtl) { + executionDeadline = submittedAt + executionTtl; + } + + if (retryLimiter.UpdateOnRetry(now, policy) && now < executionDeadline) { + queryStatus.Clear(); + // failing query is throttled for backoff period + backoff = policy.BackoffPeriod * (retryLimiter.RetryRate + 1); + owner = ""; + if (!transientIssues) { + transientIssues.ConstructInPlace(); + } + TStringBuilder builder; + builder << "Query failed with code " << NYql::NDqProto::StatusIds_StatusCode_Name(request.status_code()) + << " and will be restarted (RetryCount: " << retryLimiter.RetryCount << ")" + << " at " << now; + transientIssues->AddIssue(NYql::TIssue(builder)); + } else { + // failure query should be processed instantly + queryStatus = FederatedQuery::QueryMeta::FAILING; + backoff = TDuration::Zero(); + TStringBuilder builder; + builder << "Query failed with code " << NYql::NDqProto::StatusIds_StatusCode_Name(request.status_code()); + if (policy.RetryCount) { + builder << " (" << retryLimiter.LastError << ")"; + } + builder << " at " << now; + + // in case of problems with finalization, do not change the issues + if (query.meta().status() == FederatedQuery::QueryMeta::FAILING || query.meta().status() == FederatedQuery::QueryMeta::ABORTING_BY_SYSTEM || query.meta().status() == FederatedQuery::QueryMeta::ABORTING_BY_USER) { + if (issues) { + transientIssues->AddIssues(*issues); + } + transientIssues->AddIssue(NYql::TIssue(builder)); + } else { + if (!issues) { + issues.ConstructInPlace(); + } + auto issue = NYql::TIssue(builder); + if (query.issue().size() > 0 && request.issues().empty()) { + NYql::TIssues queryIssues; + NYql::IssuesFromMessage(query.issue(), queryIssues); + for (auto& subIssue : queryIssues) { + issue.AddSubIssue(MakeIntrusive(subIssue)); + } + } + if (transientIssues) { + for (auto& subIssue : *transientIssues) { + issue.AddSubIssue(MakeIntrusive(subIssue)); + } + transientIssues.Clear(); + } + issues->AddIssue(issue); + } + } + CPS_LOG_AS_D(*actorSystem, "PingTaskRequest (resign): " << (!policyFound ? " DEFAULT POLICY" : "") << (owner ? " FAILURE " : " ") << NYql::NDqProto::StatusIds_StatusCode_Name(request.status_code()) << " " << retryLimiter.RetryCount << " " << retryLimiter.RetryCounterUpdatedAt << " " << backoff); + } + + if (queryStatus) { + query.mutable_meta()->set_status(*queryStatus); + job.mutable_query_meta()->set_status(*queryStatus); + } + + if (request.status_code() != NYql::NDqProto::StatusIds::UNSPECIFIED) { + internal.set_status_code(request.status_code()); + } + + if (request.pending_status_code() != NYql::NDqProto::StatusIds::UNSPECIFIED) { + internal.set_pending_status_code(request.pending_status_code()); + } + + if (issues) { + NYql::IssuesToMessage(*issues, query.mutable_issue()); + NYql::IssuesToMessage(*issues, job.mutable_issue()); + } + + if (transientIssues) { + AddTransientIssues(query.mutable_transient_issue(), std::move(*transientIssues)); + } + + if (request.internal_issues().size()) { + *internal.mutable_internal_issue() = request.internal_issues(); + } + + if (request.statistics()) { + TString statistics = request.statistics(); + if (request.flat_stats_size() == 0) { + internal.clear_statistics(); + // TODO: remove once V1 and V2 stats go the same way + PackStatisticsToProtobuf(*internal.mutable_statistics(), statistics, TInstant::Now() - NProtoInterop::CastFromProto(job.meta().created_at())); + } + + // global dumpRawStatistics will be removed with YQv1 + if (!Config->Proto.GetDumpRawStatistics() && !request.dump_raw_statistics()) { + try { + statistics = GetPrettyStatistics(statistics); + } catch (const std::exception&) { + CPS_LOG_AS_E(*actorSystem, "Error on statistics prettification: " << CurrentExceptionMessage()); + } + } + *query.mutable_statistics()->mutable_json() = statistics; + *job.mutable_statistics()->mutable_json() = statistics; + } + + if (request.current_load()) { + internal.set_current_load(request.current_load()); + } + + if (request.timeline()) { + internal.set_timeline(request.timeline()); + } + + if (request.flat_stats_size() != 0) { + internal.clear_statistics(); + auto stats = DeserializeFlatStats(request.flat_stats()); + PackStatisticsToProtobuf(*internal.mutable_statistics(), stats, TInstant::Now() - NProtoInterop::CastFromProto(job.meta().created_at())); + } + + if (!request.result_set_meta().empty()) { + // we will overwrite result_set_meta's COMPLETELY + *query.mutable_result_set_meta() = request.result_set_meta(); + *job.mutable_result_set_meta() = request.result_set_meta(); + } + + if (request.ast()) { + query.mutable_ast()->set_data(request.ast()); + job.mutable_ast()->set_data(request.ast()); + } + + if (request.plan()) { + query.mutable_plan()->set_json(request.plan()); + job.mutable_plan()->set_json(request.plan()); + } + + if (request.ast_compressed().data()) { + internal.mutable_ast_compressed()->set_method(request.ast_compressed().method()); + internal.mutable_ast_compressed()->set_data(request.ast_compressed().data()); + // todo: keep AST compressed in JobInternal + // job.mutable_ast()->set_data(request.ast()); + } + + if (request.plan_compressed().data()) { + internal.mutable_plan_compressed()->set_method(request.plan_compressed().method()); + internal.mutable_plan_compressed()->set_data(request.plan_compressed().data()); + // todo: keep plan compressed in JobInternal + // job.mutable_plan()->set_json(request.plan()); + } + + if (request.has_started_at()) { + *query.mutable_meta()->mutable_started_at() = request.started_at(); + *job.mutable_query_meta()->mutable_started_at() = request.started_at(); + } + + if (request.has_finished_at()) { + *query.mutable_meta()->mutable_finished_at() = request.finished_at(); + *job.mutable_query_meta()->mutable_finished_at() = request.finished_at(); + if (!query.meta().has_started_at()) { + *query.mutable_meta()->mutable_started_at() = request.finished_at(); + *job.mutable_query_meta()->mutable_started_at() = request.finished_at(); + } + } + + if (IsTerminalStatus(query.meta().status()) && query.content().automatic()) { + *query.mutable_meta()->mutable_expire_at() = NProtoInterop::CastToProto(expireAt); + *job.mutable_query_meta()->mutable_expire_at() = NProtoInterop::CastToProto(expireAt); + *job.mutable_expire_at() = NProtoInterop::CastToProto(expireAt); + } + + if (query.meta().status() == FederatedQuery::QueryMeta::COMPLETED) { + *query.mutable_meta()->mutable_result_expire_at() = request.deadline(); + } + + if (request.state_load_mode()) { + internal.set_state_load_mode(request.state_load_mode()); + if (request.state_load_mode() == FederatedQuery::FROM_LAST_CHECKPOINT) { // Saved checkpoint + query.mutable_meta()->set_has_saved_checkpoints(true); + } + } + + if (request.has_disposition()) { + *internal.mutable_disposition() = request.disposition(); + } + + if (request.status() && IsTerminalStatus(request.status())) { + internal.clear_created_topic_consumers(); + // internal.clear_dq_graph(); keep for debug + internal.clear_dq_graph_index(); + // internal.clear_execution_id(); keep for debug + // internal.clear_operation_id(); keep for debug + } + + if (!request.created_topic_consumers().empty()) { + std::set mergedConsumers; + for (auto&& c : *internal.mutable_created_topic_consumers()) { + mergedConsumers.emplace(std::move(c)); + } + for (const auto& c : request.created_topic_consumers()) { + mergedConsumers.emplace(c); + } + internal.clear_created_topic_consumers(); + for (auto&& c : mergedConsumers) { + *internal.add_created_topic_consumers() = std::move(c); + } + } + + if (!request.execution_id().empty()) { + internal.set_execution_id(request.execution_id()); + } + + if (!request.operation_id().empty()) { + internal.set_operation_id(request.operation_id()); + } + + if (!request.dq_graph().empty()) { + *internal.mutable_dq_graph() = request.dq_graph(); + } + + if (!request.dq_graph_compressed().empty()) { + *internal.mutable_dq_graph_compressed() = request.dq_graph_compressed(); + } + + if (request.dq_graph_index()) { + internal.set_dq_graph_index(request.dq_graph_index()); + } + + if (request.has_resources()) { + *internal.mutable_resources() = request.resources(); + } + + const auto maxRequestSize = Config->Proto.GetMaxRequestSize(); + if (job.ByteSizeLong() > maxRequestSize) { + ythrow NYql::TCodeLineException(TIssuesIds::BAD_REQUEST) << "Job proto exceeded the size limit: " << job.ByteSizeLong() << " of " << maxRequestSize << " " << TSizeFormatPrinter(job).ToString(); + } + + if (query.ByteSizeLong() > maxRequestSize) { + ythrow NYql::TCodeLineException(TIssuesIds::BAD_REQUEST) << "Query proto exceeded the size limit: " << query.ByteSizeLong() << " of " << maxRequestSize << " " << TSizeFormatPrinter(query).ToString(); + } + + if (internal.ByteSizeLong() > maxRequestSize) { + ythrow NYql::TCodeLineException(TIssuesIds::BAD_REQUEST) << "QueryInternal proto exceeded the size limit: " << internal.ByteSizeLong() << " of " << maxRequestSize << " " << TSizeFormatPrinter(internal).ToString(); + } + + finalStatus->Status = query.meta().status(); + finalStatus->QueryType = query.content().type(); + finalStatus->StatusCode = internal.status_code(); + finalStatus->CloudId = internal.cloud_id(); + finalStatus->JobId = job.meta().id(); + NYql::IssuesFromMessage(query.issue(), finalStatus->Issues); + NYql::IssuesFromMessage(query.transient_issue(), finalStatus->TransientIssues); +} + +void TControlPlaneStorageBase::FillQueryStatistics( + const std::shared_ptr& finalStatus, const FederatedQuery::Query& query, + const FederatedQuery::Internal::QueryInternal& internal, const TRetryLimiter& retryLimiter) const +{ + finalStatus->FinalStatistics = ExtractStatisticsFromProtobuf(internal.statistics()); + finalStatus->FinalStatistics.push_back(std::make_pair("IsAutomatic", query.content().automatic())); + if (query.content().name().Contains("DataLens YQ query")) { + finalStatus->FinalStatistics.push_back(std::make_pair("IsDataLens", 1)); + } else if (query.content().name().Contains("Audit-trails")) { + finalStatus->FinalStatistics.push_back(std::make_pair("IsAuditTrails", 1)); + } else if (query.content().name().Contains("Query from YDB SDK")) { + finalStatus->FinalStatistics.push_back(std::make_pair("IsSDK", 1)); + } + finalStatus->FinalStatistics.push_back(std::make_pair("RetryCount", retryLimiter.RetryCount)); + finalStatus->FinalStatistics.push_back(std::make_pair("RetryRate", retryLimiter.RetryRate * 100)); + finalStatus->FinalStatistics.push_back(std::make_pair("Load", internal.current_load())); +} + void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev) { TInstant startTime = TInstant::Now(); @@ -635,20 +643,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvPingTaskReq requestCounters.IncInFly(); requestCounters.Common->RequestBytes->Add(ev->Get()->GetByteSize()); const TString queryId = request.query_id().value(); - const TString owner = request.owner_id(); - const TInstant deadline = NProtoInterop::CastFromProto(request.deadline()); - const TString tenant = request.tenant(); CPS_LOG_T("PingTaskRequest: {" << request.DebugString() << "}"); - NYql::TIssues issues = ValidatePingTask(scope, queryId, owner, deadline, Config->ResultSetsTtl); - - auto tenantInfo = ev->Get()->TenantInfo; - if (tenantInfo && tenantInfo->TenantState.Value(tenant, TenantState::Active) == TenantState::Idle) { - issues.AddIssue("Tenant is idle, no processing is allowed"); - } - - if (issues) { + if (const auto& issues = ValidateRequest(ev)) { CPS_LOG_W("PingTaskRequest: {" << request.DebugString() << "} validation FAILED: " << issues.ToOneLineString()); const TDuration delta = TInstant::Now() - startTime; SendResponseIssues(ev->Sender, issues, ev->Cookie, delta, requestCounters); @@ -660,10 +658,8 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvPingTaskReq std::shared_ptr finalStatus = std::make_shared(); auto pingTaskParams = DoesPingTaskUpdateQueriesTable(request) ? - ConstructHardPingTask(request, response, YdbConnection->TablePathPrefix, Config->AutomaticQueriesTtl, - Config->TaskLeaseTtl, Config->RetryPolicies, Counters.Counters, Config->Proto.GetMaxRequestSize(), - Config->Proto.GetDumpRawStatistics(), finalStatus, requestCounters.Common) : - ConstructSoftPingTask(request, response, YdbConnection->TablePathPrefix, Config->TaskLeaseTtl, requestCounters.Common); + ConstructHardPingTask(request, response, finalStatus, requestCounters.Common) : + ConstructSoftPingTask(request, response, requestCounters.Common); auto debugInfo = Config->Proto.GetEnableDebugMode() ? std::make_shared() : TDebugInfoPtr{}; auto result = ReadModifyWrite(pingTaskParams.Query, pingTaskParams.Params, pingTaskParams.Prepare, requestCounters, debugInfo); auto prepare = [response] { return *response; }; @@ -696,7 +692,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvPingTaskReq }); } -void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvFinalStatusReport::TPtr& ev) { +void TControlPlaneStorageBase::Handle(TEvControlPlaneStorage::TEvFinalStatusReport::TPtr& ev) { const auto& event = *ev->Get(); if (!IsTerminalStatus(event.Status)) { return; @@ -719,11 +715,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvFinalStatus Counters.GetFinalStatusCounters(event.CloudId, event.Scope)->IncByStatus(event.Status); - Statistics statistics{event.Statistics}; + TStatistics statistics{event.Statistics}; LOG_YQ_AUDIT_SERVICE_INFO("FinalStatus: cloud id: [" << event.CloudId << "], scope: [" << event.Scope << "], query id: [" << event.QueryId << "], job id: [" << event.JobId << "], query type: [" << FederatedQuery::QueryContent::QueryType_Name(event.QueryType) << "], " << statistics << ", " << "status: " << FederatedQuery::QueryMeta::ComputeStatus_Name(event.Status)); } - } // NFq diff --git a/ydb/core/fq/libs/control_plane_storage/internal/task_result_write.cpp b/ydb/core/fq/libs/control_plane_storage/internal/task_result_write.cpp index c2a39dbfe05b..469f8eab3230 100644 --- a/ydb/core/fq/libs/control_plane_storage/internal/task_result_write.cpp +++ b/ydb/core/fq/libs/control_plane_storage/internal/task_result_write.cpp @@ -1,7 +1,19 @@ #include "utils.h" +#include + namespace NFq { +NYql::TIssues TControlPlaneStorageBase::ValidateRequest(TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr& ev) const { + const auto& request = ev->Get()->Request; + return ValidateWriteResultData( + request.result_id().value(), + request.result_set(), + NProtoInterop::CastFromProto(request.deadline()), + Config->ResultSetsTtl + ); +} + void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr& ev) { TInstant startTime = TInstant::Now(); @@ -17,11 +29,9 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvWriteResult const Ydb::ResultSet& resultSet = request.result_set(); const int byteSize = resultSet.ByteSize(); - CPS_LOG_T("WriteResultDataRequest: " << resultId << " " << resultSetId << " " << startRowId << " " << resultSet.ByteSize() << " " << deadline); - NYql::TIssues issues = ValidateWriteResultData(resultId, resultSet, deadline, Config->ResultSetsTtl); - if (issues) { + if (const auto& issues = ValidateRequest(ev)) { CPS_LOG_D("WriteResultDataRequest, validation failed: " << resultId << " " << resultSetId << " " << startRowId << " " << resultSet.DebugString() << " " << deadline << " error: " << issues.ToString()); const TDuration delta = TInstant::Now() - startTime; SendResponseIssues(ev->Sender, issues, ev->Cookie, delta, requestCounters); diff --git a/ydb/core/fq/libs/control_plane_storage/internal/utils.cpp b/ydb/core/fq/libs/control_plane_storage/internal/utils.cpp index 3c9281e86eda..ab7c8a3617e0 100644 --- a/ydb/core/fq/libs/control_plane_storage/internal/utils.cpp +++ b/ydb/core/fq/libs/control_plane_storage/internal/utils.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -143,15 +144,15 @@ std::vector GetMeteringRecords(const TString& statistics, bool billable } auto now = Now(); - result.emplace_back(TBillRecord() + result.emplace_back(NKikimr::TBillRecord() .Id(jobId + "_i") .Schema("yq.ingress.v1") .FolderId(TScope(scope).ParseFolder()) .SourceWt(now) .SourceId(sourceId) - .Usage(TBillRecord::TUsage() - .Type(TBillRecord::TUsage::EType::Delta) - .Unit(TBillRecord::TUsage::EUnit::MByte) + .Usage(NKikimr::TBillRecord::TUsage() + .Type(NKikimr::TBillRecord::TUsage::EType::Delta) + .Unit(NKikimr::TBillRecord::TUsage::EUnit::MByte) .Quantity(ingressMBytes) .Start(now) .Finish(now) @@ -311,8 +312,8 @@ void PackStatisticsToProtobuf(google::protobuf::RepeatedPtrField& statsProto) { - StatsValuesList statPairs; +TStatsValuesList ExtractStatisticsFromProtobuf(const google::protobuf::RepeatedPtrField& statsProto) { + TStatsValuesList statPairs; statPairs.reserve(statsProto.size()); for (const auto& stat : statsProto) { statPairs.emplace_back(stat.name(), stat.value()); @@ -320,7 +321,7 @@ StatsValuesList ExtractStatisticsFromProtobuf(const google::protobuf::RepeatedPt return statPairs; } -TStringBuilder& operator<<(TStringBuilder& builder, const Statistics& statistics) { +TStringBuilder& operator<<(TStringBuilder& builder, const TStatistics& statistics) { bool first = true; builder << '{'; for (const auto& [field, value] : statistics.Stats) { diff --git a/ydb/core/fq/libs/control_plane_storage/internal/utils.h b/ydb/core/fq/libs/control_plane_storage/internal/utils.h index 778b960081d4..90adcfd1f31c 100644 --- a/ydb/core/fq/libs/control_plane_storage/internal/utils.h +++ b/ydb/core/fq/libs/control_plane_storage/internal/utils.h @@ -2,12 +2,15 @@ #include +#include + #include #include #include -#include +#include +#include #include namespace NFq { @@ -41,17 +44,17 @@ void PackStatisticsToProtobuf(google::protobuf::RepeatedPtrField>; +using TStatsValuesList = std::vector>; -StatsValuesList ExtractStatisticsFromProtobuf(const google::protobuf::RepeatedPtrField& statsProto); +TStatsValuesList ExtractStatisticsFromProtobuf(const google::protobuf::RepeatedPtrField& statsProto); -struct Statistics { +struct TStatistics { operator bool() const noexcept { return !Stats.empty(); } - const StatsValuesList& Stats; + const TStatsValuesList& Stats; }; -TStringBuilder& operator<<(TStringBuilder& builder, const Statistics& statistics); +TStringBuilder& operator<<(TStringBuilder& builder, const TStatistics& statistics); void AddTransientIssues(::google::protobuf::RepeatedPtrField< ::Ydb::Issue::IssueMessage>* protoIssues, NYql::TIssues&& issues); diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage.cpp b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage.cpp index 84b6f687f84d..7413bee14834 100644 --- a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage.cpp +++ b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage.cpp @@ -32,6 +32,20 @@ TYdbSdkRetryPolicy::TPtr MakeCreateSchemaRetryPolicy() { } // namespace +TControlPlaneStorageBase::TControlPlaneStorageBase( + const NConfig::TControlPlaneStorageConfig& config, + const NYql::TS3GatewayConfig& s3Config, + const NConfig::TCommonConfig& common, + const NConfig::TComputeConfig& computeConfig, + const ::NMonitoring::TDynamicCounterPtr& counters, + const TString& tenantName) + : TBase(config, s3Config, common, computeConfig) + , Counters(counters, *Config) + , FailedStatusCodeCounters(MakeIntrusive("FinalFailedStatusCode", counters->GetSubgroup("component", "QueryDiagnostic"))) + , TenantName(tenantName) +{ +} + void TYdbControlPlaneStorageActor::Bootstrap() { CPS_LOG_I("Starting ydb control plane storage service. Actor id: " << SelfId()); NLwTraceMonPage::ProbeRegistry().AddProbesList(LWTRACE_GET_PROBES(YQ_CONTROL_PLANE_STORAGE_PROVIDER)); @@ -323,7 +337,7 @@ void TYdbControlPlaneStorageActor::AfterTablesCreated() { // Schedule(TDuration::Zero(), new NActors::TEvents::TEvWakeup()); } -bool TControlPlaneStorageUtils::IsSuperUser(const TString& user) +bool TControlPlaneStorageUtils::IsSuperUser(const TString& user) const { return AnyOf(Config->Proto.GetSuperUsers(), [&user](const auto& superUser) { return superUser == user; diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp index 3eb1d7cee0bc..a643a520f32e 100644 --- a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp +++ b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp @@ -8,6 +8,46 @@ namespace NFq { +NYql::TIssues TControlPlaneStorageBase::ValidateRequest(TEvControlPlaneStorage::TEvCreateBindingRequest::TPtr& ev) const +{ + NYql::TIssues issues = ValidateBinding(ev); + + const auto& event = *ev->Get(); + const auto& permissions = GetCreateBindingPerimssions(event); + if (event.Request.content().acl().visibility() == FederatedQuery::Acl::SCOPE && !permissions.Check(TPermissions::MANAGE_PUBLIC)) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::ACCESS_DENIED, "Permission denied to create a binding with these parameters. Please receive a permission yq.resources.managePublic")); + } + + return issues; +} + +TPermissions TControlPlaneStorageBase::GetCreateBindingPerimssions(const TEvControlPlaneStorage::TEvCreateBindingRequest& event) const +{ + TPermissions permissions = Config->Proto.GetEnablePermissions() + ? event.Permissions + : TPermissions{TPermissions::MANAGE_PUBLIC}; + if (IsSuperUser(event.User)) { + permissions.SetAll(); + } + return permissions; +} + +std::pair TControlPlaneStorageBase::GetCreateBindingProtos( + const FederatedQuery::CreateBindingRequest& request, const TString& cloudId, const TString& user, TInstant startTime) const +{ + const TString& bindingId = GetEntityIdAsString(Config->IdsPrefix, EEntityType::BINDING); + + FederatedQuery::Binding binding; + FederatedQuery::BindingContent& content = *binding.mutable_content(); + content = request.content(); + *binding.mutable_meta() = CreateCommonMeta(bindingId, user, startTime, InitialRevision); + + FederatedQuery::Internal::BindingInternal bindingInternal; + bindingInternal.set_cloud_id(cloudId); + + return {binding, bindingInternal}; +} + void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateBindingRequest::TPtr& ev) { TInstant startTime = TInstant::Now(); @@ -19,28 +59,21 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateBindi requestCounters.Common->RequestBytes->Add(event.GetByteSize()); const TString user = event.User; const TString token = event.Token; - TPermissions permissions = Config->Proto.GetEnablePermissions() - ? event.Permissions - : TPermissions{TPermissions::MANAGE_PUBLIC}; - if (IsSuperUser(user)) { - permissions.SetAll(); - } const FederatedQuery::CreateBindingRequest& request = event.Request; - const TString bindingId = GetEntityIdAsString(Config->IdsPrefix, EEntityType::BINDING); int byteSize = request.ByteSize(); const TString connectionId = request.content().connection_id(); const TString idempotencyKey = request.idempotency_key(); + const auto [binding, bindingInternal] = GetCreateBindingProtos(request, cloudId, user, startTime); + const auto& content = binding.content(); + const TString& bindingId = binding.meta().id(); + CPS_LOG_T(MakeLogPrefix(scope, user, bindingId) << "CreateBindingRequest: " << NKikimr::MaskTicket(token) << " " << request.DebugString()); - NYql::TIssues issues = ValidateBinding(ev); - if (request.content().acl().visibility() == FederatedQuery::Acl::SCOPE && !permissions.Check(TPermissions::MANAGE_PUBLIC)) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::ACCESS_DENIED, "Permission denied to create a binding with these parameters. Please receive a permission yq.resources.managePublic")); - } - if (issues) { + if (const auto& issues = ValidateRequest(ev)) { CPS_LOG_D(MakeLogPrefix(scope, user, bindingId) << "CreateBindingRequest, validation failed: " << NKikimr::MaskTicket(token) << " " @@ -52,14 +85,6 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateBindi return; } - FederatedQuery::Binding binding; - FederatedQuery::BindingContent& content = *binding.mutable_content(); - content = request.content(); - *binding.mutable_meta() = CreateCommonMeta(bindingId, user, startTime, InitialRevision); - - FederatedQuery::Internal::BindingInternal bindingInternal; - bindingInternal.set_cloud_id(cloudId); - std::shared_ptr>> response = std::make_shared>>(); response->first.set_binding_id(bindingId); response->second.After.ConstructInPlace().CopyFrom(binding); @@ -111,7 +136,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateBindi scope, connectionId, "Connection " + connectionId + " does not exist or permission denied. Please check the id connection or your access rights", - permissions, + GetCreateBindingPerimssions(event), user, content.acl().visibility(), YdbConnection->TablePathPrefix); @@ -122,7 +147,6 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateBindi user, YdbConnection->TablePathPrefix); - TVector validators; if (idempotencyKey) { validators.push_back(CreateIdempotencyKeyValidator(scope, idempotencyKey, response, YdbConnection->TablePathPrefix, requestCounters.Common->ParseProtobufError)); diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp index 6a8692038993..3a0021deaa48 100644 --- a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp +++ b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp @@ -30,6 +30,46 @@ void PrepareSensitiveFields(::FederatedQuery::Connection& connection, bool extra } +NYql::TIssues TControlPlaneStorageBase::ValidateRequest(TEvControlPlaneStorage::TEvCreateConnectionRequest::TPtr& ev) const +{ + NYql::TIssues issues = ValidateConnection(ev); + + const auto& event = *ev->Get(); + const auto& permissions = GetCreateConnectionPerimssions(event); + if (event.Request.content().acl().visibility() == FederatedQuery::Acl::SCOPE && !permissions.Check(TPermissions::MANAGE_PUBLIC)) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::ACCESS_DENIED, "Permission denied to create a connection with these parameters. Please receive a permission yq.resources.managePublic")); + } + + return issues; +} + +TPermissions TControlPlaneStorageBase::GetCreateConnectionPerimssions(const TEvControlPlaneStorage::TEvCreateConnectionRequest& event) const +{ + TPermissions permissions = Config->Proto.GetEnablePermissions() + ? event.Permissions + : TPermissions{TPermissions::MANAGE_PUBLIC}; + if (IsSuperUser(event.User)) { + permissions.SetAll(); + } + return permissions; +} + +std::pair TControlPlaneStorageBase::GetCreateConnectionProtos( + const FederatedQuery::CreateConnectionRequest& request, const TString& cloudId, const TString& user, TInstant startTime) const +{ + const TString& connectionId = GetEntityIdAsString(Config->IdsPrefix, EEntityType::CONNECTION); + + FederatedQuery::Connection connection; + FederatedQuery::ConnectionContent& content = *connection.mutable_content(); + content = request.content(); + *connection.mutable_meta() = CreateCommonMeta(connectionId, user, startTime, InitialRevision); + + FederatedQuery::Internal::ConnectionInternal connectionInternal; + connectionInternal.set_cloud_id(cloudId); + + return {connection, connectionInternal}; +} + void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateConnectionRequest::TPtr& ev) { TInstant startTime = TInstant::Now(); @@ -43,25 +83,18 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateConne const TString user = event.User; const TString token = event.Token; const int byteSize = request.ByteSize(); - TPermissions permissions = Config->Proto.GetEnablePermissions() - ? event.Permissions - : TPermissions{TPermissions::MANAGE_PUBLIC}; - if (IsSuperUser(user)) { - permissions.SetAll(); - } const TString idempotencyKey = request.idempotency_key(); - const TString connectionId = GetEntityIdAsString(Config->IdsPrefix, EEntityType::CONNECTION); + + const auto [connection, connectionInternal] = GetCreateConnectionProtos(request, cloudId, user, startTime); + const auto& content = connection.content(); + const TString& connectionId = connection.meta().id(); CPS_LOG_T(MakeLogPrefix(scope, user, connectionId) << "CreateConnectionRequest: " << NKikimr::MaskTicket(token) << " " << request.DebugString()); - NYql::TIssues issues = ValidateConnection(ev); - if (request.content().acl().visibility() == FederatedQuery::Acl::SCOPE && !permissions.Check(TPermissions::MANAGE_PUBLIC)) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::ACCESS_DENIED, "Permission denied to create a connection with these parameters. Please receive a permission yq.resources.managePublic")); - } - if (issues) { + if (const auto& issues = ValidateRequest(ev)) { CPS_LOG_D(MakeLogPrefix(scope, user, connectionId) << "CreateConnectionRequest, validation failed: " << NKikimr::MaskTicket(token) << " " @@ -73,14 +106,6 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateConne return; } - FederatedQuery::Connection connection; - FederatedQuery::ConnectionContent& content = *connection.mutable_content(); - content = request.content(); - *connection.mutable_meta() = CreateCommonMeta(connectionId, user, startTime, InitialRevision); - - FederatedQuery::Internal::ConnectionInternal connectionInternal; - connectionInternal.set_cloud_id(cloudId); - std::shared_ptr>> response = std::make_shared>>(); response->first.set_connection_id(connectionId); response->second.After.ConstructInPlace().CopyFrom(connection); @@ -141,7 +166,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateConne auto overridBindingValidator = CreateConnectionOverrideBindingValidator( scope, content.name(), - permissions, + GetCreateConnectionPerimssions(event), user, YdbConnection->TablePathPrefix); validators.push_back(overridBindingValidator); diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_impl.h b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_impl.h index 6a5ebf9b75b2..203c115f0120 100644 --- a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_impl.h +++ b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_impl.h @@ -8,6 +8,7 @@ #include "request_validators.h" #include "util.h" #include "validators.h" +#include #include #include @@ -313,10 +314,10 @@ class TControlPlaneStorageUtils { /* * Utility */ - bool IsSuperUser(const TString& user); + bool IsSuperUser(const TString& user) const; template - NYql::TIssues ValidateConnection(T& ev, bool passwordRequired = true) + NYql::TIssues ValidateConnection(T& ev, bool passwordRequired = true) const { return ::NFq::ValidateConnection(ev, Config->Proto.GetMaxRequestSize(), Config->AvailableConnections, Config->Proto.GetDisableCurrentIam(), @@ -324,19 +325,19 @@ class TControlPlaneStorageUtils { } template - NYql::TIssues ValidateBinding(T& ev) + NYql::TIssues ValidateBinding(T& ev) const { return ::NFq::ValidateBinding(ev, Config->Proto.GetMaxRequestSize(), Config->AvailableBindings, Config->GeneratorPathsLimit); } template - NYql::TIssues ValidateQuery(const T& ev) + NYql::TIssues ValidateQuery(const T& ev) const { return ::NFq::ValidateQuery(ev, Config->Proto.GetMaxRequestSize()); } template - NYql::TIssues ValidateEvent(const P& ev) + NYql::TIssues ValidateEvent(const P& ev) const { return ::NFq::ValidateEvent

(ev, Config->Proto.GetMaxRequestSize()); } @@ -371,10 +372,8 @@ class TControlPlaneStorageUtils { static constexpr int64_t InitialRevision = 1; }; -class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped, - public TDbRequester, - public TControlPlaneStorageUtils -{ +class TControlPlaneStorageBase : public TControlPlaneStorageUtils { +protected: enum ERequestTypeScope { RTS_CREATE_QUERY, RTS_LIST_QUERIES, @@ -559,6 +558,10 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped cacheVal; ScopeCounters.Get(key, &cacheVal); @@ -581,13 +584,217 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped GetCreateQueryProtos( + const FederatedQuery::CreateQueryRequest& request, const TString& user, TInstant startTime) const; + + FederatedQuery::Internal::QueryInternal GetQueryInternalProto( + const FederatedQuery::CreateQueryRequest& request, const TString& cloudId, const TString& token, + const TMaybe& quotas) const; + + void FillConnectionsAndBindings( + FederatedQuery::Internal::QueryInternal& queryInternal, FederatedQuery::QueryContent::QueryType queryType, + const TVector& allConnections, + const THashMap& visibleConnections, + const THashMap& visibleBindings) const; + + // Describe query request + + NYql::TIssues ValidateRequest(TEvControlPlaneStorage::TEvDescribeQueryRequest::TPtr& ev) const; + + void FillDescribeQueryResult( + FederatedQuery::DescribeQueryResult& result, FederatedQuery::Internal::QueryInternal internal, + const TString& user, TPermissions permissions) const; + + // Get result data request + + NYql::TIssues ValidateRequest(TEvControlPlaneStorage::TEvGetResultDataRequest::TPtr& ev) const; + + TPermissions GetResultDataReadPerimssions(const TEvControlPlaneStorage::TEvGetResultDataRequest& event) const; + + // Create connection request + + NYql::TIssues ValidateRequest(TEvControlPlaneStorage::TEvCreateConnectionRequest::TPtr& ev) const; + + TPermissions GetCreateConnectionPerimssions(const TEvControlPlaneStorage::TEvCreateConnectionRequest& event) const; + + std::pair GetCreateConnectionProtos( + const FederatedQuery::CreateConnectionRequest& request, const TString& cloudId, const TString& user, TInstant startTime) const; + + // Create binding request + + NYql::TIssues ValidateRequest(TEvControlPlaneStorage::TEvCreateBindingRequest::TPtr& ev) const; + + TPermissions GetCreateBindingPerimssions(const TEvControlPlaneStorage::TEvCreateBindingRequest& event) const; + + std::pair GetCreateBindingProtos( + const FederatedQuery::CreateBindingRequest& request, const TString& cloudId, const TString& user, TInstant startTime) const; + + // Write result data request + + NYql::TIssues ValidateRequest(TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr& ev) const; + + // Get task request + + NYql::TIssues ValidateRequest(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev) const; + + void FillGetTaskResult(Fq::Private::GetTaskResult& result, const TVector& tasks) const; + + // Ping task request + + struct TFinalStatus { + FederatedQuery::QueryMeta::ComputeStatus Status = FederatedQuery::QueryMeta::COMPUTE_STATUS_UNSPECIFIED; + NYql::NDqProto::StatusIds::StatusCode StatusCode = NYql::NDqProto::StatusIds::UNSPECIFIED; + FederatedQuery::QueryContent::QueryType QueryType = FederatedQuery::QueryContent::QUERY_TYPE_UNSPECIFIED; + NYql::TIssues Issues; + NYql::TIssues TransientIssues; + TStatsValuesList FinalStatistics; + TString CloudId; + TString JobId; + }; + + NYql::TIssues ValidateRequest(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev) const; + + void UpdateTaskInfo( + NActors::TActorSystem* actorSystem, Fq::Private::PingTaskRequest& request, const std::shared_ptr& finalStatus, FederatedQuery::Query& query, + FederatedQuery::Internal::QueryInternal& internal, FederatedQuery::Job& job, TString& owner, + TRetryLimiter& retryLimiter, TDuration& backoff, TInstant& expireAt) const; + + void FillQueryStatistics( + const std::shared_ptr& finalStatus, const FederatedQuery::Query& query, + const FederatedQuery::Internal::QueryInternal& internal, const TRetryLimiter& retryLimiter) const; + + void Handle(TEvControlPlaneStorage::TEvFinalStatusReport::TPtr& ev); + + // Node health check + + NYql::TIssues ValidateRequest(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev) const; + +protected: + // Should not be used from callbacks + template + void SendResponseIssues(TActorId sender, const NYql::TIssues& issues, ui64 cookie, const TDuration& delta, TRequestCounters requestCounters) { + std::unique_ptr event(new T{issues}); + requestCounters.Common->ResponseBytes->Add(event->GetByteSize()); + TActivationContext::Send(sender, std::move(event), 0, cookie); + requestCounters.DecInFly(); + requestCounters.IncError(); + requestCounters.Common->LatencyMs->Collect(delta.MilliSeconds()); + } + + template + TFuture SendResponse(const TString& name, + NActors::TActorSystem* actorSystem, + const TAsyncStatus& status, + TActorId self, + const RequestEventPtr& ev, + const TInstant& startTime, + const TRequestCounters& requestCounters, + const std::function::Type()>& prepare, + TDebugInfoPtr debugInfo) + { + return status.Apply([=, requestCounters=requestCounters](const auto& future) mutable { + NYql::TIssues internalIssues; + NYql::TIssues issues; + Result result; + typename TPrepareResponseResultType::TResponseAuditDetails auditDetails; // void* for nonauditable events + + try { + TStatus status = future.GetValue(); + if (status.IsSuccess()) { + if constexpr (ResponseEvent::Auditable) { + auto p = prepare(); + result = std::move(p.first); + auditDetails = std::move(p.second); + } else { + result = prepare(); + } + } else { + issues.AddIssues(NYdb::NAdapters::ToYqlIssues(status.GetIssues())); + internalIssues.AddIssues(NYdb::NAdapters::ToYqlIssues(status.GetIssues())); + } + } catch (const NYql::TCodeLineException& exception) { + NYql::TIssue issue = MakeErrorIssue(exception.Code, exception.GetRawMessage()); + issues.AddIssue(issue); + NYql::TIssue internalIssue = MakeErrorIssue(exception.Code, CurrentExceptionMessage()); + internalIssues.AddIssue(internalIssue); + } catch (const std::exception& exception) { + NYql::TIssue issue = MakeErrorIssue(TIssuesIds::INTERNAL_ERROR, exception.what()); + issues.AddIssue(issue); + NYql::TIssue internalIssue = MakeErrorIssue(TIssuesIds::INTERNAL_ERROR, CurrentExceptionMessage()); + internalIssues.AddIssue(internalIssue); + } catch (...) { + NYql::TIssue issue = MakeErrorIssue(TIssuesIds::INTERNAL_ERROR, CurrentExceptionMessage()); + issues.AddIssue(issue); + NYql::TIssue internalIssue = MakeErrorIssue(TIssuesIds::INTERNAL_ERROR, CurrentExceptionMessage()); + internalIssues.AddIssue(internalIssue); + } + + const auto& request = ev->Get()->Request; + size_t responseByteSize = 0; + if (issues) { + CPS_LOG_AS_W(*actorSystem, name << ": {" << TrimForLogs(request.DebugString()) << "} ERROR: " << internalIssues.ToOneLineString()); + auto event = std::make_unique(issues); + event->DebugInfo = debugInfo; + responseByteSize = event->GetByteSize(); + actorSystem->Send(new IEventHandle(ev->Sender, self, event.release(), 0, ev->Cookie)); + requestCounters.IncError(); + for (const auto& issue : issues) { + NYql::WalkThroughIssues(issue, true, [&requestCounters](const NYql::TIssue& err, ui16 level) { + Y_UNUSED(level); + requestCounters.Common->Issues->GetCounter(ToString(err.GetCode()), true)->Inc(); + }); + } + } else { + CPS_LOG_AS_T(*actorSystem, name << ": {" << TrimForLogs(result.DebugString()) << "} SUCCESS"); + std::unique_ptr event; + if constexpr (ResponseEvent::Auditable) { + event = std::make_unique(result, auditDetails); + } else { + event = std::make_unique(result); + } + event->DebugInfo = debugInfo; + responseByteSize = event->GetByteSize(); + actorSystem->Send(new IEventHandle(ev->Sender, self, event.release(), 0, ev->Cookie)); + requestCounters.IncOk(); + } + requestCounters.DecInFly(); + requestCounters.Common->ResponseBytes->Add(responseByteSize); + TDuration delta = TInstant::Now() - startTime; + requestCounters.Common->LatencyMs->Collect(delta.MilliSeconds()); + return MakeFuture(!issues); + }); + } + + static ui64 GetExecutionLimitMills(FederatedQuery::QueryContent::QueryType queryType, const TMaybe& quotas); +}; + +class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped, + public TDbRequester, + public TControlPlaneStorageBase +{ + using TBase = TControlPlaneStorageBase; ::NFq::TYqSharedResources::TPtr YqSharedResources; NKikimr::TYdbCredentialsProviderFactory CredProviderFactory; - TString TenantName; // Query Quota THashMap QueryQuotas; @@ -607,12 +814,9 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped("FinalFailedStatusCode", counters->GetSubgroup("component", "QueryDiagnostic"))) + : TBase(config, s3Config, common, computeConfig, counters, tenantName) , YqSharedResources(yqSharedResources) , CredProviderFactory(credProviderFactory) - , TenantName(tenantName) { } @@ -655,7 +859,7 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped void HandleRateLimiterImpl(TEventPtr& ev); @@ -737,90 +939,6 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped - TFuture SendResponse(const TString& name, - NActors::TActorSystem* actorSystem, - const TAsyncStatus& status, - TActorId self, - const RequestEventPtr& ev, - const TInstant& startTime, - const TRequestCounters& requestCounters, - const std::function::Type()>& prepare, - TDebugInfoPtr debugInfo) - { - return status.Apply([=, requestCounters=requestCounters](const auto& future) mutable { - NYql::TIssues internalIssues; - NYql::TIssues issues; - Result result; - typename TPrepareResponseResultType::TResponseAuditDetails auditDetails; // void* for nonauditable events - - try { - TStatus status = future.GetValue(); - if (status.IsSuccess()) { - if constexpr (ResponseEvent::Auditable) { - auto p = prepare(); - result = std::move(p.first); - auditDetails = std::move(p.second); - } else { - result = prepare(); - } - } else { - issues.AddIssues(NYdb::NAdapters::ToYqlIssues(status.GetIssues())); - internalIssues.AddIssues(NYdb::NAdapters::ToYqlIssues(status.GetIssues())); - } - } catch (const NYql::TCodeLineException& exception) { - NYql::TIssue issue = MakeErrorIssue(exception.Code, exception.GetRawMessage()); - issues.AddIssue(issue); - NYql::TIssue internalIssue = MakeErrorIssue(exception.Code, CurrentExceptionMessage()); - internalIssues.AddIssue(internalIssue); - } catch (const std::exception& exception) { - NYql::TIssue issue = MakeErrorIssue(TIssuesIds::INTERNAL_ERROR, exception.what()); - issues.AddIssue(issue); - NYql::TIssue internalIssue = MakeErrorIssue(TIssuesIds::INTERNAL_ERROR, CurrentExceptionMessage()); - internalIssues.AddIssue(internalIssue); - } catch (...) { - NYql::TIssue issue = MakeErrorIssue(TIssuesIds::INTERNAL_ERROR, CurrentExceptionMessage()); - issues.AddIssue(issue); - NYql::TIssue internalIssue = MakeErrorIssue(TIssuesIds::INTERNAL_ERROR, CurrentExceptionMessage()); - internalIssues.AddIssue(internalIssue); - } - - const auto& request = ev->Get()->Request; - size_t responseByteSize = 0; - if (issues) { - CPS_LOG_AS_W(*actorSystem, name << ": {" << TrimForLogs(request.DebugString()) << "} ERROR: " << internalIssues.ToOneLineString()); - auto event = std::make_unique(issues); - event->DebugInfo = debugInfo; - responseByteSize = event->GetByteSize(); - actorSystem->Send(new IEventHandle(ev->Sender, self, event.release(), 0, ev->Cookie)); - requestCounters.IncError(); - for (const auto& issue : issues) { - NYql::WalkThroughIssues(issue, true, [&requestCounters](const NYql::TIssue& err, ui16 level) { - Y_UNUSED(level); - requestCounters.Common->Issues->GetCounter(ToString(err.GetCode()), true)->Inc(); - }); - } - } else { - CPS_LOG_AS_T(*actorSystem, name << ": {" << TrimForLogs(result.DebugString()) << "} SUCCESS"); - std::unique_ptr event; - if constexpr (ResponseEvent::Auditable) { - event = std::make_unique(result, auditDetails); - } else { - event = std::make_unique(result); - } - event->DebugInfo = debugInfo; - responseByteSize = event->GetByteSize(); - actorSystem->Send(new IEventHandle(ev->Sender, self, event.release(), 0, ev->Cookie)); - requestCounters.IncOk(); - } - requestCounters.DecInFly(); - requestCounters.Common->ResponseBytes->Add(responseByteSize); - TDuration delta = TInstant::Now() - startTime; - requestCounters.Common->LatencyMs->Collect(delta.MilliSeconds()); - return MakeFuture(!issues); - }); - } - template TFuture SendResponseTuple(const TString& name, NActors::TActorSystem* actorSystem, @@ -891,20 +1009,6 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped - void SendResponseIssues(const TActorId sender, - const NYql::TIssues& issues, - ui64 cookie, - const TDuration& delta, - TRequestCounters requestCounters) { - std::unique_ptr event(new T{issues}); - requestCounters.Common->ResponseBytes->Add(event->GetByteSize()); - Send(sender, event.release(), 0, cookie); - requestCounters.DecInFly(); - requestCounters.IncError(); - requestCounters.Common->LatencyMs->Collect(delta.MilliSeconds()); - } - struct TPickTaskParams { TString ReadQuery; TParams ReadParams; @@ -921,9 +1025,20 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped& validators = {}, TTxSettings transactionMode = TTxSettings::SerializableRW()); - ui64 GetExecutionLimitMills( - FederatedQuery::QueryContent_QueryType queryType, - const TMaybe& quotas); + struct TPingTaskParams { + TString Query; + TParams Params; + const std::function(const std::vector&)> Prepare; + std::shared_ptr> MeteringRecords; + }; + + TPingTaskParams ConstructHardPingTask( + const Fq::Private::PingTaskRequest& request, std::shared_ptr response, + const std::shared_ptr& finalStatus, const TRequestCommonCountersPtr& commonCounters) const; + + TPingTaskParams ConstructSoftPingTask( + const Fq::Private::PingTaskRequest& request, std::shared_ptr response, + const TRequestCommonCountersPtr& commonCounters) const; }; } diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp index ae9a1a30e912..84a3f5c70a22 100644 --- a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp +++ b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp @@ -54,36 +54,17 @@ FederatedQuery::IamAuth::IdentityCase GetIamAuth(const FederatedQuery::Connectio namespace NFq { -void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQueryRequest::TPtr& ev) +NYql::TIssues TControlPlaneStorageBase::ValidateRequest(TEvControlPlaneStorage::TEvCreateQueryRequest::TPtr& ev) const { - TInstant startTime = TInstant::Now(); const TEvControlPlaneStorage::TEvCreateQueryRequest& event = *ev->Get(); - const TString cloudId = event.CloudId; const FederatedQuery::CreateQueryRequest& request = event.Request; - const FederatedQuery::Internal::ComputeDatabaseInternal& computeDatabase = event.ComputeDatabase; - ui64 resultLimit = 0; - if (event.Quotas) { - if (auto it = event.Quotas->find(QUOTA_QUERY_RESULT_LIMIT); it != event.Quotas->end()) { - resultLimit = it->second.Limit.Value; - } - } - auto queryType = request.content().type(); - ui64 executionLimitMills = GetExecutionLimitMills(queryType, event.Quotas); - const TString scope = event.Scope; - TRequestCounters requestCounters = Counters.GetCounters(cloudId, scope, RTS_CREATE_QUERY, RTC_CREATE_QUERY); - requestCounters.IncInFly(); - requestCounters.Common->RequestBytes->Add(event.GetByteSize()); - const TString user = event.User; - const TString token = event.Token; + TPermissions permissions = Config->Proto.GetEnablePermissions() ? event.Permissions : TPermissions{TPermissions::QUERY_INVOKE | TPermissions::MANAGE_PUBLIC}; - if (IsSuperUser(user)) { + if (IsSuperUser(event.User)) { permissions.SetAll(); } - const size_t byteSize = request.ByteSizeLong(); - const TString queryId = GetEntityIdAsString(Config->IdsPrefix, EEntityType::QUERY); - CPS_LOG_T("CreateQueryRequest: {" << request.DebugString() << "} " << MakeUserInfo(user, token)); NYql::TIssues issues = ValidateQuery(ev); if (request.execute_mode() != FederatedQuery::SAVE && !permissions.Check(TPermissions::QUERY_INVOKE)) { @@ -97,10 +78,9 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "Streaming disposition \"from_last_checkpoint\" is not allowed in CreateQuery request")); } - auto tenant = ev->Get()->TenantInfo->Assign(cloudId, scope, request.content().type(), TenantName); - if (event.Quotas) { TQuotaMap::const_iterator it = event.Quotas->end(); + const auto queryType = request.content().type(); if (queryType == FederatedQuery::QueryContent::ANALYTICS) { it = event.Quotas->find(QUOTA_ANALYTICS_COUNT_LIMIT); } else if (queryType == FederatedQuery::QueryContent::STREAMING) { @@ -120,24 +100,22 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "VCPU rate limit can't be less than zero")); } - if (issues) { - CPS_LOG_W("CreateQueryRequest: {" << request.DebugString() << "} " << MakeUserInfo(user, token) << "validation FAILED: " << issues.ToOneLineString()); - const TDuration delta = TInstant::Now() - startTime; - SendResponseIssues(ev->Sender, issues, ev->Cookie, delta, requestCounters); - LWPROBE(CreateQueryRequest, scope, user, delta, byteSize, false); - return; - } - - const TString idempotencyKey = request.idempotency_key(); - const TString jobId = request.execute_mode() == FederatedQuery::SAVE ? "" : GetEntityIdAsString(Config->IdsPrefix, EEntityType::JOB); + return issues; +} +std::pair TControlPlaneStorageBase::GetCreateQueryProtos( + const FederatedQuery::CreateQueryRequest& request, const TString& user, TInstant startTime) const +{ FederatedQuery::Query query; FederatedQuery::QueryContent& content = *query.mutable_content() = request.content(); FederatedQuery::QueryMeta& meta = *query.mutable_meta(); + + const TString queryId = GetEntityIdAsString(Config->IdsPrefix, EEntityType::QUERY); FederatedQuery::CommonMeta& common = *meta.mutable_common() = CreateCommonMeta(queryId, user, startTime, InitialRevision); meta.set_execute_mode(request.execute_mode()); meta.set_status(request.execute_mode() == FederatedQuery::SAVE ? FederatedQuery::QueryMeta::COMPLETED : FederatedQuery::QueryMeta::STARTING); + const TString jobId = request.execute_mode() == FederatedQuery::SAVE ? "" : GetEntityIdAsString(Config->IdsPrefix, EEntityType::JOB); FederatedQuery::Job job; if (request.execute_mode() != FederatedQuery::SAVE) { meta.set_last_job_query_revision(InitialRevision); @@ -154,6 +132,112 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery *job.mutable_parameters() = content.parameters(); } + return {query, job}; +} + +FederatedQuery::Internal::QueryInternal TControlPlaneStorageBase::GetQueryInternalProto( + const FederatedQuery::CreateQueryRequest& request, const TString& cloudId, const TString& token, + const TMaybe& quotas) const +{ + FederatedQuery::Internal::QueryInternal queryInternal; + if (!Config->Proto.GetDisableCurrentIam()) { + queryInternal.set_token(token); + } + + queryInternal.set_cloud_id(cloudId); + queryInternal.set_state_load_mode(FederatedQuery::StateLoadMode::EMPTY); + queryInternal.mutable_disposition()->CopyFrom(request.disposition()); + ui64 resultLimit = 0; + if (quotas) { + if (const auto it = quotas->find(QUOTA_QUERY_RESULT_LIMIT); it != quotas->end()) { + resultLimit = it->second.Limit.Value; + } + } + queryInternal.set_result_limit(resultLimit); + *queryInternal.mutable_execution_ttl() = NProtoInterop::CastToProto(TDuration::MilliSeconds(GetExecutionLimitMills(request.content().type(), quotas))); + + return queryInternal; +} + +void TControlPlaneStorageBase::FillConnectionsAndBindings( + FederatedQuery::Internal::QueryInternal& queryInternal, FederatedQuery::QueryContent::QueryType queryType, + const TVector& allConnections, + const THashMap& visibleConnections, + const THashMap& visibleBindings) const +{ + TSet disabledConnections; + for (const auto& connection : allConnections) { + const auto connectionCase = connection.content().setting().connection_case(); + if (!Config->AvailableConnections.contains(connectionCase)) { + disabledConnections.insert(connection.meta().id()); + continue; + } + if ((queryType == FederatedQuery::QueryContent::STREAMING) && !Config->AvailableStreamingConnections.contains(connectionCase)) { + disabledConnections.insert(connection.meta().id()); + continue; + } + if (GetIamAuth(connection) == FederatedQuery::IamAuth::kCurrentIam && Config->Proto.GetDisableCurrentIam()) { + disabledConnections.insert(connection.meta().id()); + continue; + } + } + + TSet connectionIds; + for (const auto& [_, connection] : visibleConnections) { + if (disabledConnections.contains(connection.meta().id())) { + continue; + } + *queryInternal.add_connection() = connection; + connectionIds.insert(connection.meta().id()); + } + + for (const auto& [_, binding] : visibleBindings) { + if (!Config->AvailableBindings.contains(binding.content().setting().binding_case())) { + continue; + } + if (disabledConnections.contains(binding.content().connection_id())) { + continue; + } + + *queryInternal.add_binding() = binding; + if (!connectionIds.contains(binding.content().connection_id())) { + ythrow NYql::TCodeLineException(TIssuesIds::BAD_REQUEST) << "Unable to resolve connection for binding " << binding.meta().id() << ", name " << binding.content().name() << ", connection id " << binding.content().connection_id(); + } + } +} + +void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQueryRequest::TPtr& ev) +{ + TInstant startTime = TInstant::Now(); + const TEvControlPlaneStorage::TEvCreateQueryRequest& event = *ev->Get(); + const TString cloudId = event.CloudId; + const FederatedQuery::CreateQueryRequest& request = event.Request; + const FederatedQuery::Internal::ComputeDatabaseInternal& computeDatabase = event.ComputeDatabase; + const auto queryType = request.content().type(); + const TString scope = event.Scope; + TRequestCounters requestCounters = Counters.GetCounters(cloudId, scope, RTS_CREATE_QUERY, RTC_CREATE_QUERY); + requestCounters.IncInFly(); + requestCounters.Common->RequestBytes->Add(event.GetByteSize()); + const TString user = event.User; + const TString token = event.Token; + const size_t byteSize = request.ByteSizeLong(); + CPS_LOG_T("CreateQueryRequest: {" << request.DebugString() << "} " << MakeUserInfo(user, token)); + + auto tenant = ev->Get()->TenantInfo->Assign(cloudId, scope, queryType, TenantName); + + if (const auto& issues = ValidateRequest(ev)) { + CPS_LOG_W("CreateQueryRequest: {" << request.DebugString() << "} " << MakeUserInfo(user, token) << "validation FAILED: " << issues.ToOneLineString()); + const TDuration delta = TInstant::Now() - startTime; + SendResponseIssues(ev->Sender, issues, ev->Cookie, delta, requestCounters); + LWPROBE(CreateQueryRequest, scope, user, delta, byteSize, false); + return; + } + + const TString idempotencyKey = request.idempotency_key(); + const auto [query, job] = GetCreateQueryProtos(request, user, startTime); + const TString queryId = query.meta().common().id(); + const TString jobId = job.meta().id(); + std::shared_ptr>> response = std::make_shared>>(); response->first.set_query_id(queryId); response->second.CloudId = cloudId; @@ -179,7 +263,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery ); } - auto prepareParams = [=, as=TActivationContext::ActorSystem(), commonCounters=requestCounters.Common](const std::vector& resultSets) mutable { + auto prepareParams = [=, as=TActivationContext::ActorSystem(), commonCounters=requestCounters.Common, quotas=event.Quotas](const std::vector& resultSets) mutable { const size_t countSets = (idempotencyKey ? 1 : 0) + (request.execute_mode() != FederatedQuery::SAVE ? 2 : 0); if (resultSets.size() != countSets) { ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to " << countSets << " but equal " << resultSets.size() << ". Please contact internal support"; @@ -197,63 +281,18 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery } } - FederatedQuery::Internal::QueryInternal queryInternal; - if (!Config->Proto.GetDisableCurrentIam()) { - queryInternal.set_token(token); - } - - queryInternal.set_cloud_id(cloudId); - queryInternal.set_state_load_mode(FederatedQuery::StateLoadMode::EMPTY); - queryInternal.mutable_disposition()->CopyFrom(request.disposition()); - queryInternal.set_result_limit(resultLimit); - *queryInternal.mutable_execution_ttl() = NProtoInterop::CastToProto(TDuration::MilliSeconds(executionLimitMills)); + auto queryInternal = GetQueryInternalProto(request, cloudId, token, quotas); if (request.execute_mode() != FederatedQuery::SAVE) { // TODO: move to run actor priority selection *queryInternal.mutable_compute_connection() = computeDatabase.connection(); - TSet disabledConnections; - for (const auto& connection: GetEntities(resultSets[resultSets.size() - 2], CONNECTION_COLUMN_NAME, Config->Proto.GetIgnorePrivateSources(), commonCounters)) { - auto connectionCase = connection.content().setting().connection_case(); - if (!Config->AvailableConnections.contains(connectionCase)) { - disabledConnections.insert(connection.meta().id()); - continue; - } - if ((queryType == FederatedQuery::QueryContent::STREAMING) && !Config->AvailableStreamingConnections.contains(connectionCase)) { - disabledConnections.insert(connection.meta().id()); - continue; - } - - if (GetIamAuth(connection) == FederatedQuery::IamAuth::kCurrentIam && Config->Proto.GetDisableCurrentIam()) { - disabledConnections.insert(connection.meta().id()); - continue; - } - } - - TSet connectionIds; - auto connections = GetEntitiesWithVisibilityPriority(resultSets[resultSets.size() - 2], CONNECTION_COLUMN_NAME, Config->Proto.GetIgnorePrivateSources(), commonCounters); - for (const auto& [_, connection]: connections) { - if (disabledConnections.contains(connection.meta().id())) { - continue; - } - *queryInternal.add_connection() = connection; - connectionIds.insert(connection.meta().id()); - } - - auto bindings = GetEntitiesWithVisibilityPriority(resultSets[resultSets.size() - 1], BINDING_COLUMN_NAME, Config->Proto.GetIgnorePrivateSources(), commonCounters); - for (const auto& [_, binding]: bindings) { - if (!Config->AvailableBindings.contains(binding.content().setting().binding_case())) { - continue; - } - - if (disabledConnections.contains(binding.content().connection_id())) { - continue; - } - - *queryInternal.add_binding() = binding; - if (!connectionIds.contains(binding.content().connection_id())) { - ythrow NYql::TCodeLineException(TIssuesIds::BAD_REQUEST) << "Unable to resolve connection for binding " << binding.meta().id() << ", name " << binding.content().name() << ", connection id " << binding.content().connection_id(); - } - } + FillConnectionsAndBindings( + queryInternal, + queryType, + GetEntities(resultSets[resultSets.size() - 2], CONNECTION_COLUMN_NAME, Config->Proto.GetIgnorePrivateSources(), commonCounters), + GetEntitiesWithVisibilityPriority(resultSets[resultSets.size() - 2], CONNECTION_COLUMN_NAME, Config->Proto.GetIgnorePrivateSources(), commonCounters), + GetEntitiesWithVisibilityPriority(resultSets[resultSets.size() - 1], BINDING_COLUMN_NAME, Config->Proto.GetIgnorePrivateSources(), commonCounters) + ); } if (query.ByteSizeLong() > Config->Proto.GetMaxRequestSize()) { @@ -510,6 +549,73 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListQueries }); } +NYql::TIssues TControlPlaneStorageBase::ValidateRequest(TEvControlPlaneStorage::TEvDescribeQueryRequest::TPtr& ev) const +{ + return ValidateEvent(ev); +} + +void TControlPlaneStorageBase::FillDescribeQueryResult( + FederatedQuery::DescribeQueryResult& result, FederatedQuery::Internal::QueryInternal internal, + const TString& user, TPermissions permissions) const +{ + const auto lastJobId = result.query().meta().last_job_id(); + result.mutable_query()->mutable_meta()->set_last_job_id(lastJobId + "-" + result.query().meta().common().id()); + + permissions = Config->Proto.GetEnablePermissions() + ? permissions + : TPermissions{TPermissions::VIEW_PUBLIC | TPermissions::VIEW_AST | TPermissions::VIEW_QUERY_TEXT}; + if (IsSuperUser(user)) { + permissions.SetAll(); + } + + const auto queryVisibility = result.query().content().acl().visibility(); + const auto queryUser = result.query().meta().common().created_by(); + const bool hasViewAccess = HasViewAccess(permissions, queryVisibility, queryUser, user); + if (!hasViewAccess) { + ythrow NYql::TCodeLineException(TIssuesIds::ACCESS_DENIED) << "Query does not exist or permission denied. Please check the id of the query or your access rights"; + } + + // decompress plan + if (internal.plan_compressed().data()) { // todo: remove this if after migration + TCompressor compressor(internal.plan_compressed().method()); + result.mutable_query()->mutable_plan()->set_json(compressor.Decompress(internal.plan_compressed().data())); + if (result.query().ByteSizeLong() > GRPC_MESSAGE_SIZE_LIMIT) { + if (result.query().plan().json().size() > 1000) { + // modifing plan this way should definitely reduce query msg size + result.mutable_query()->mutable_plan()->set_json(TStringBuilder() << "Message is too big: " << result.query().ByteSizeLong() << " bytes, dropping plan of size " << result.query().plan().json().size() << " bytes"); + } + } + } + + auto timeline = internal.timeline(); + if (timeline) { + result.mutable_query()->mutable_timeline()->set_svg(timeline); + } + + if (!permissions.Check(TPermissions::VIEW_AST)) { + result.mutable_query()->clear_ast(); + } else { + // decompress AST + if (internal.ast_compressed().data()) { // todo: remove this if after migration + TCompressor compressor(internal.ast_compressed().method()); + result.mutable_query()->mutable_ast()->set_data(compressor.Decompress(internal.ast_compressed().data())); + } + if (result.query().ByteSizeLong() > GRPC_MESSAGE_SIZE_LIMIT) { + if (result.query().ast().data().size() > 1000) { + // modifing AST this way should definitely reduce query msg size + result.mutable_query()->mutable_ast()->set_data(TStringBuilder() << "Message is too big: " << result.query().ByteSizeLong() << " bytes, dropping AST of size " << result.query().ast().data().size() << " bytes"); + } + } + } + if (!permissions.Check(TPermissions::VIEW_QUERY_TEXT)) { + result.mutable_query()->mutable_content()->clear_text(); + } + + if (result.query().ByteSizeLong() > GRPC_MESSAGE_SIZE_LIMIT) { + ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Resulting query of size " << result.query().ByteSizeLong() << " bytes is too big"; + } +} + void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeQueryRequest::TPtr& ev) { TInstant startTime = TInstant::Now(); @@ -521,20 +627,13 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeQue requestCounters.Common->RequestBytes->Add(event.GetByteSize()); const TString user = event.User; const TString token = event.Token; - TPermissions permissions = Config->Proto.GetEnablePermissions() - ? event.Permissions - : TPermissions{TPermissions::VIEW_PUBLIC | TPermissions::VIEW_AST | TPermissions::VIEW_QUERY_TEXT}; - if (IsSuperUser(user)) { - permissions.SetAll(); - } const FederatedQuery::DescribeQueryRequest& request = event.Request; const TString queryId = request.query_id(); const int byteSize = request.ByteSize(); CPS_LOG_T("DescribeQueryRequest: {" << request.DebugString() << "} " << MakeUserInfo(user, token)); - NYql::TIssues issues = ValidateEvent(ev); - if (issues) { + if (const auto& issues = ValidateRequest(ev)) { CPS_LOG_W("DescribeQueryRequest: {" << request.DebugString() << "} " << MakeUserInfo(user, token) << "validation FAILED: " << issues.ToOneLineString()); const TDuration delta = TInstant::Now() - startTime; SendResponseIssues(ev->Sender, issues, ev->Cookie, delta, requestCounters); @@ -553,7 +652,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeQue const auto query = queryBuilder.Build(); auto debugInfo = Config->Proto.GetEnableDebugMode() ? std::make_shared() : TDebugInfoPtr{}; auto [result, resultSets] = Read(query.Sql, query.Params, requestCounters, debugInfo); - auto prepare = [resultSets=resultSets, user, permissions, commonCounters=requestCounters.Common] { + auto prepare = [this, resultSets=resultSets, user, permissions=event.Permissions, commonCounters=requestCounters.Common] { if (resultSets->size() != 1) { ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 1 but equal " << resultSets->size() << ". Please contact internal support"; } @@ -569,61 +668,13 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeQue ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support"; } - const auto lastJobId = result.query().meta().last_job_id(); - result.mutable_query()->mutable_meta()->set_last_job_id(lastJobId + "-" + result.query().meta().common().id()); - - const auto queryVisibility = result.query().content().acl().visibility(); - const auto queryUser = result.query().meta().common().created_by(); - const bool hasViewAccess = HasViewAccess(permissions, queryVisibility, queryUser, user); - if (!hasViewAccess) { - ythrow NYql::TCodeLineException(TIssuesIds::ACCESS_DENIED) << "Query does not exist or permission denied. Please check the id of the query or your access rights"; - } - FederatedQuery::Internal::QueryInternal internal; if (!internal.ParseFromString(*parser.ColumnParser(INTERNAL_COLUMN_NAME).GetOptionalString())) { commonCounters->ParseProtobufError->Inc(); ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query internal. Please contact internal support"; } - // decompress plan - if (internal.plan_compressed().data()) { // todo: remove this if after migration - TCompressor compressor(internal.plan_compressed().method()); - result.mutable_query()->mutable_plan()->set_json(compressor.Decompress(internal.plan_compressed().data())); - if (result.query().ByteSizeLong() > GRPC_MESSAGE_SIZE_LIMIT) { - if (result.query().plan().json().size() > 1000) { - // modifing plan this way should definitely reduce query msg size - result.mutable_query()->mutable_plan()->set_json(TStringBuilder() << "Message is too big: " << result.query().ByteSizeLong() << " bytes, dropping plan of size " << result.query().plan().json().size() << " bytes"); - } - } - } - - auto timeline = internal.timeline(); - if (timeline) { - result.mutable_query()->mutable_timeline()->set_svg(timeline); - } - - if (!permissions.Check(TPermissions::VIEW_AST)) { - result.mutable_query()->clear_ast(); - } else { - // decompress AST - if (internal.ast_compressed().data()) { // todo: remove this if after migration - TCompressor compressor(internal.ast_compressed().method()); - result.mutable_query()->mutable_ast()->set_data(compressor.Decompress(internal.ast_compressed().data())); - } - if (result.query().ByteSizeLong() > GRPC_MESSAGE_SIZE_LIMIT) { - if (result.query().ast().data().size() > 1000) { - // modifing AST this way should definitely reduce query msg size - result.mutable_query()->mutable_ast()->set_data(TStringBuilder() << "Message is too big: " << result.query().ByteSizeLong() << " bytes, dropping AST of size " << result.query().ast().data().size() << " bytes"); - } - } - } - if (!permissions.Check(TPermissions::VIEW_QUERY_TEXT)) { - result.mutable_query()->mutable_content()->clear_text(); - } - - if (result.query().ByteSizeLong() > GRPC_MESSAGE_SIZE_LIMIT) { - ythrow NYql::TCodeLineException(TIssuesIds::INTERNAL_ERROR) << "Resulting query of size " << result.query().ByteSizeLong() << " bytes is too big"; - } + FillDescribeQueryResult(result, internal, user, permissions); return result; }; @@ -1484,6 +1535,21 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer }); } +NYql::TIssues TControlPlaneStorageBase::ValidateRequest(TEvControlPlaneStorage::TEvGetResultDataRequest::TPtr& ev) const +{ + return ValidateEvent(ev); +} + +TPermissions TControlPlaneStorageBase::GetResultDataReadPerimssions(const TEvControlPlaneStorage::TEvGetResultDataRequest& event) const { + TPermissions permissions = Config->Proto.GetEnablePermissions() + ? event.Permissions + : TPermissions{TPermissions::VIEW_PUBLIC}; + if (IsSuperUser(event.User)) { + permissions.SetAll(); + } + return permissions; +} + void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetResultDataRequest::TPtr& ev) { TInstant startTime = TInstant::Now(); @@ -1501,17 +1567,11 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetResultDa const TString queryId = request.query_id(); const int byteSize = event.Request.ByteSize(); const TString token = event.Token; - TPermissions permissions = Config->Proto.GetEnablePermissions() - ? event.Permissions - : TPermissions{TPermissions::VIEW_PUBLIC}; - if (IsSuperUser(user)) { - permissions.SetAll(); - } + const TPermissions permissions = GetResultDataReadPerimssions(event); const int64_t limit = request.limit(); CPS_LOG_T("GetResultDataRequest: {" << request.DebugString() << "} " << MakeUserInfo(user, token)); - NYql::TIssues issues = ValidateEvent(ev); - if (issues) { + if (const auto& issues = ValidateRequest(ev)) { CPS_LOG_W("GetResultDataRequest: {" << request.DebugString() << "} " << MakeUserInfo(user, token) << "validation FAILED: " << issues.ToOneLineString()); const TDuration delta = TInstant::Now() - startTime; SendResponseIssues(ev->Sender, issues, ev->Cookie, delta, requestCounters); @@ -1851,10 +1911,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeJob }); } -ui64 TYdbControlPlaneStorageActor::GetExecutionLimitMills( - FederatedQuery::QueryContent_QueryType queryType, - const TMaybe& quotas) { - +ui64 TControlPlaneStorageBase::GetExecutionLimitMills(FederatedQuery::QueryContent::QueryType queryType, const TMaybe& quotas) { if (!quotas) { return 0; } diff --git a/ydb/core/fq/libs/init/init.cpp b/ydb/core/fq/libs/init/init.cpp index 3bdc089b3677..d18203c641a2 100644 --- a/ydb/core/fq/libs/init/init.cpp +++ b/ydb/core/fq/libs/init/init.cpp @@ -77,14 +77,21 @@ void Init( const auto clientCounters = yqCounters->GetSubgroup("subsystem", "ClientMetrics"); if (protoConfig.GetControlPlaneStorage().GetEnabled()) { + const auto counters = yqCounters->GetSubgroup("subsystem", "ControlPlaneStorage"); auto controlPlaneStorage = protoConfig.GetControlPlaneStorage().GetUseInMemory() - ? NFq::CreateInMemoryControlPlaneStorageServiceActor(protoConfig.GetControlPlaneStorage()) + ? NFq::CreateInMemoryControlPlaneStorageServiceActor( + protoConfig.GetControlPlaneStorage(), + protoConfig.GetGateways().GetS3(), + protoConfig.GetCommon(), + protoConfig.GetCompute(), + counters, + tenant) : NFq::CreateYdbControlPlaneStorageServiceActor( protoConfig.GetControlPlaneStorage(), protoConfig.GetGateways().GetS3(), protoConfig.GetCommon(), protoConfig.GetCompute(), - yqCounters->GetSubgroup("subsystem", "ControlPlaneStorage"), + counters, yqSharedResources, NKikimr::CreateYdbCredentialsProviderFactory, tenant); diff --git a/ydb/tests/fq/control_plane_storage/in_memory_control_plane_storage_ut.cpp b/ydb/tests/fq/control_plane_storage/in_memory_control_plane_storage_ut.cpp index c7266237bd7f..de0bdb869de4 100644 --- a/ydb/tests/fq/control_plane_storage/in_memory_control_plane_storage_ut.cpp +++ b/ydb/tests/fq/control_plane_storage/in_memory_control_plane_storage_ut.cpp @@ -1,3 +1,5 @@ +#include + #include #include @@ -12,74 +14,111 @@ #include #include +#include + namespace NFq { using namespace NActors; using namespace NKikimr; +using namespace NFqRun; +using namespace NKikimrRun; namespace { -//////////////////////////////////////////////////////////////////////////////// +class TTestFuxture : public NUnitTest::TBaseFixture { +protected: + using TBase = NUnitTest::TBaseFixture; -using TRuntimePtr = std::unique_ptr; + static constexpr TDuration WAIT_TIMEOUT = TDuration::Seconds(15); -struct TTestBootstrap { - NConfig::TControlPlaneStorageConfig Config; - TRuntimePtr Runtime; +public: + void SetUp(NUnitTest::TTestContext& ctx) override { + TBase::SetUp(ctx); - const TDuration RequestTimeout = TDuration::Seconds(10); + SetupSignalActions(); + EnableYDBBacktraceFormat(); - TTestBootstrap() - { - Runtime = PrepareTestActorRuntime(); - } + TFqSetupSettings settings; + settings.VerboseLevel = TFqSetupSettings::EVerbose::Max; - std::pair CreateQuery() - { - TActorId sender = Runtime->AllocateEdgeActor(); - FederatedQuery::CreateQueryRequest proto; + settings.LogConfig.SetDefaultLevel(NLog::EPriority::PRI_WARN); + ModifyLogPriorities({{NKikimrServices::EServiceKikimr::YQ_CONTROL_PLANE_STORAGE, NLog::EPriority::PRI_DEBUG}}, settings.LogConfig); - FederatedQuery::QueryContent& content = *proto.mutable_content(); - content.set_name("my_query_1"); - content.set_text("SELECT 1;"); - content.set_type(FederatedQuery::QueryContent::ANALYTICS); - content.mutable_acl()->set_visibility(FederatedQuery::Acl::SCOPE); + auto& cpStorageConfig = *settings.FqConfig.MutableControlPlaneStorage(); + cpStorageConfig.SetEnabled(true); + cpStorageConfig.SetUseInMemory(true); - auto request = std::make_unique("yandexcloud://my_cloud_1/my_folder_1", proto, "user@staff", "", "mock_cloud", TPermissions{}, TQuotaMap{}, std::make_shared(), FederatedQuery::Internal::ComputeDatabaseInternal{}); - Runtime->Send(new IEventHandle(ControlPlaneStorageServiceActorId(), sender, request.release())); + auto& privateApiConfig = *settings.FqConfig.MutablePrivateApi(); + privateApiConfig.SetEnabled(true); + privateApiConfig.SetLoopback(true); + + settings.FqConfig.SetEnableDynamicNameservice(true); + settings.FqConfig.MutableControlPlaneProxy()->SetEnabled(true); + settings.FqConfig.MutableDbPool()->SetEnabled(true); + settings.FqConfig.MutableNodesManager()->SetEnabled(true); + settings.FqConfig.MutablePendingFetcher()->SetEnabled(true); + settings.FqConfig.MutablePrivateProxy()->SetEnabled(true); + settings.FqConfig.MutableGateways()->SetEnabled(true); + settings.FqConfig.MutableResourceManager()->SetEnabled(true); + settings.FqConfig.MutableCommon()->SetIdsPrefix("ut"); + + FqSetup = std::make_unique(settings); + } - TAutoPtr handle; - TEvControlPlaneStorage::TEvCreateQueryResponse* event = Runtime->GrabEdgeEvent(handle); - return {event->Result, event->Issues}; +protected: + static void CheckSuccess(const TRequestResult& result) { + UNIT_ASSERT_VALUES_EQUAL_C(result.Status, Ydb::StatusIds::SUCCESS, result.Issues.ToOneLineString()); } -private: - TRuntimePtr PrepareTestActorRuntime() { - TRuntimePtr runtime(new TTestBasicRuntime(1)); - runtime->SetLogPriority(NKikimrServices::YQ_CONTROL_PLANE_STORAGE, NLog::PRI_DEBUG); + TExecutionMeta WaitQueryExecution(const TString& queryId, TDuration timeout = WAIT_TIMEOUT) const { + using EStatus = FederatedQuery::QueryMeta; - auto controlPlaneProxy = CreateInMemoryControlPlaneStorageServiceActor(Config); + const TInstant start = TInstant::Now(); + while (TInstant::Now() - start <= timeout) { + TExecutionMeta meta; + CheckSuccess(FqSetup->DescribeQuery(queryId, meta)); - runtime->AddLocalService( - ControlPlaneStorageServiceActorId(), - TActorSetupCmd(controlPlaneProxy, TMailboxType::Simple, 0)); + if (!IsIn({EStatus::FAILED, EStatus::COMPLETED, EStatus::ABORTED_BY_USER, EStatus::ABORTED_BY_SYSTEM}, meta.Status)) { + Cerr << "Wait query execution " << TInstant::Now() - start << ": " << EStatus::ComputeStatus_Name(meta.Status) << "\n"; + Sleep(TDuration::Seconds(1)); + continue; + } - SetupTabletServices(*runtime); + UNIT_ASSERT_C(meta.Status == EStatus::COMPLETED, "issues: " << meta.Issues.ToOneLineString() << ", transient issues: " << meta.TransientIssues.ToOneLineString()); + return meta; + } - return runtime; + UNIT_ASSERT_C(false, "Waiting query execution timeout. Spent time " << TInstant::Now() - start << " exceeds limit " << timeout); + return {}; } + +protected: + std::unique_ptr FqSetup; }; -} +} // anonymous namespace + +Y_UNIT_TEST_SUITE(InMemoryControlPlaneStorage) { + Y_UNIT_TEST_F(ExecuteSimpleQuery, TTestFuxture) { + TString queryId; + CheckSuccess(FqSetup->StreamRequest({ + .Query = "SELECT 42 AS result_value" + }, queryId)); + UNIT_ASSERT(queryId); -Y_UNIT_TEST_SUITE(CreateQueryRequest) { - Y_UNIT_TEST(ShouldCreateSimpleQuery) - { - TTestBootstrap bootstrap; - const auto [result, issues] = bootstrap.CreateQuery(); - UNIT_ASSERT(!issues); + const auto meta = WaitQueryExecution(queryId); + UNIT_ASSERT_VALUES_EQUAL(meta.ResultSetSizes.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(meta.ResultSetSizes[0], 1); + + Ydb::ResultSet resultSet; + CheckSuccess(FqSetup->FetchQueryResults(queryId, 0, resultSet)); + + NYdb::TResultSetParser parser(resultSet); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnsCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(parser.RowsCount(), 1); + UNIT_ASSERT(parser.TryNextRow()); + UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("result_value").GetInt32(), 42); } } - } // NFq diff --git a/ydb/tests/fq/control_plane_storage/ya.make b/ydb/tests/fq/control_plane_storage/ya.make index 4939a08deef2..698e352f3c88 100644 --- a/ydb/tests/fq/control_plane_storage/ya.make +++ b/ydb/tests/fq/control_plane_storage/ya.make @@ -14,6 +14,7 @@ ENDIF() PEERDIR( library/cpp/retry library/cpp/testing/unittest + ydb/core/base ydb/core/external_sources ydb/core/fq/libs/actors/logging ydb/core/fq/libs/init @@ -22,6 +23,8 @@ PEERDIR( ydb/core/fq/libs/rate_limiter/events ydb/core/testlib/default ydb/library/security + ydb/tests/tools/fqrun/src + ydb/tests/tools/kqprun/runlib ) INCLUDE(${ARCADIA_ROOT}/ydb/public/tools/ydb_recipe/recipe.inc) diff --git a/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h b/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h index 684147f48f06..11fa568e61c3 100644 --- a/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h +++ b/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -31,6 +32,9 @@ #include #include #include + +#include + #include #include @@ -41,6 +45,7 @@ namespace NFq { using namespace NActors; using namespace NKikimr; +using namespace NKikimrRun; ////////////////////////////////////////////////////// @@ -96,6 +101,9 @@ struct TTestBootstrap { TTestBootstrap(std::string tablePrefix, const NConfig::TControlPlaneStorageConfig& config = {}) : Config(config) { + SetupSignalActions(); + EnableYDBBacktraceFormat(); + Cerr << "Netstat: " << Exec("netstat --all --program") << Endl; Cerr << "Process stat: " << Exec("ps aux") << Endl; Cerr << "YDB receipt endpoint: " << GetEnv("YDB_ENDPOINT") << ", database: " << GetEnv("YDB_DATABASE") << Endl;