Skip to content

Commit 27d22ba

Browse files
Hor911Hor911
authored
Publish timeline explicitly in separate field (#9223)
1 parent ee906a5 commit 27d22ba

File tree

8 files changed

+45
-13
lines changed

8 files changed

+45
-13
lines changed

‎ydb/core/fq/libs/compute/common/utils.cpp

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -433,7 +433,7 @@ void EnumeratePlans(NYson::TYsonWriter& writer, NJson::TJsonValue& value, ui32&
433433
}
434434
}
435435

436-
TString GetV1StatFromV2Plan(const TString& plan, double* cpuUsage, TString* timeline) {
436+
TString GetV1StatFromV2Plan(const TString& plan, double* cpuUsage, TString* timeline, ui64 maxTimelineSize) {
437437
TStringStream out;
438438
NYson::TYsonWriter writer(&out);
439439
writer.OnBeginMap();
@@ -478,6 +478,16 @@ TString GetV1StatFromV2Plan(const TString& plan, double* cpuUsage, TString* time
478478
TPlanVisualizer planViz;
479479
planViz.LoadPlans(plan);
480480
*timeline = planViz.PrintSvgSafe();
481+
if (maxTimelineSize && timeline->size() > maxTimelineSize) {
482+
TStringBuilder builder;
483+
builder
484+
<< "<svg width='600' height='200' xmlns='http://www.w3.org/2000/svg'>" << Endl
485+
<< " <text font-size='16px' x='20' y='40'>There is nothing wrong with the request.</text>" << Endl
486+
<< " <text font-size='16px' x='20' y='80'>Unfortunately, image size " << timeline->size() << " is too large.</text>" << Endl
487+
<< " <text font-size='16px' x='20' y='120'>It exceeds limit of " << maxTimelineSize << " and was discarded</text>" << Endl
488+
<< "</svg>" << Endl;
489+
*timeline = builder;
490+
}
481491
// remove json "timeline" field after migration
482492
writer.OnKeyedItem("timeline");
483493
writer.OnStringScalar(*timeline);
@@ -1156,7 +1166,7 @@ struct TNoneStatProcessor : IPlanStatProcessor {
11561166
return plan;
11571167
}
11581168

1159-
TString GetQueryStat(const TString&, double& cpuUsage, TString*) override {
1169+
TString GetQueryStat(const TString&, double& cpuUsage, TString*, ui64) override {
11601170
cpuUsage = 0.0;
11611171
return "";
11621172
}
@@ -1189,8 +1199,8 @@ struct TPlanStatProcessor : IPlanStatProcessor {
11891199
return plan;
11901200
}
11911201

1192-
TString GetQueryStat(const TString& plan, double& cpuUsage, TString* timeline) override {
1193-
return GetV1StatFromV2Plan(plan, &cpuUsage, timeline);
1202+
TString GetQueryStat(const TString& plan, double& cpuUsage, TString* timeline, ui64 maxtimelineSize) override {
1203+
return GetV1StatFromV2Plan(plan, &cpuUsage, timeline, maxtimelineSize);
11941204
}
11951205

11961206
TPublicStat GetPublicStat(const TString& stat) override {
@@ -1221,8 +1231,8 @@ struct TProfileStatProcessor : TPlanStatProcessor {
12211231
};
12221232

12231233
struct TProdStatProcessor : TFullStatProcessor {
1224-
TString GetQueryStat(const TString& plan, double& cpuUsage, TString* timeline) override {
1225-
return GetPrettyStatistics(GetV1StatFromV2Plan(plan, &cpuUsage, timeline));
1234+
TString GetQueryStat(const TString& plan, double& cpuUsage, TString* timeline, ui64 maxtimelineSize) override {
1235+
return GetPrettyStatistics(GetV1StatFromV2Plan(plan, &cpuUsage, timeline, maxtimelineSize));
12261236
}
12271237
};
12281238

@@ -1240,8 +1250,12 @@ std::unique_ptr<IPlanStatProcessor> CreateStatProcessor(const TString& statViewN
12401250

12411251
PingTaskRequestBuilder::PingTaskRequestBuilder(const NConfig::TCommonConfig& commonConfig, std::unique_ptr<IPlanStatProcessor>&& processor)
12421252
: Compressor(commonConfig.GetQueryArtifactsCompressionMethod(), commonConfig.GetQueryArtifactsCompressionMinSize())
1243-
, Processor(std::move(processor)), ShowQueryTimeline(commonConfig.GetShowQueryTimeline())
1244-
{}
1253+
, Processor(std::move(processor)), ShowQueryTimeline(commonConfig.GetShowQueryTimeline()), MaxQueryTimelineSize(commonConfig.GetMaxQueryTimelineSize())
1254+
{
1255+
if (!MaxQueryTimelineSize) {
1256+
MaxQueryTimelineSize = 200_KB;
1257+
}
1258+
}
12451259

12461260
Fq::Private::PingTaskRequest PingTaskRequestBuilder::Build(
12471261
const NYdb::NQuery::TExecStats& queryStats,
@@ -1307,7 +1321,7 @@ Fq::Private::PingTaskRequest PingTaskRequestBuilder::Build(const TString& queryP
13071321
CpuUsage = 0.0;
13081322
try {
13091323
TString timeline;
1310-
auto stat = Processor->GetQueryStat(plan, CpuUsage, ShowQueryTimeline ? &timeline : nullptr);
1324+
auto stat = Processor->GetQueryStat(plan, CpuUsage, ShowQueryTimeline ? &timeline : nullptr, MaxQueryTimelineSize);
13111325
pingTaskRequest.set_statistics(stat);
13121326
pingTaskRequest.set_dump_raw_statistics(true);
13131327
if (timeline) {

‎ydb/core/fq/libs/compute/common/utils.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ inline std::shared_ptr<NYdb::NTable::TTableClient> CreateNewTableClient(const TS
2929
tableSettings);
3030
}
3131

32-
TString GetV1StatFromV2Plan(const TString& plan, double* cpuUsage = nullptr, TString* timeline = nullptr);
32+
TString GetV1StatFromV2Plan(const TString& plan, double* cpuUsage = nullptr, TString* timeline = nullptr, ui64 maxTimelineSize = 0);
3333
TString GetV1StatFromV2PlanV2(const TString& plan);
3434
TString GetPrettyStatistics(const TString& statistics);
3535
THashMap<TString, i64> AggregateStats(TStringBuf plan);
@@ -56,7 +56,7 @@ struct IPlanStatProcessor {
5656
virtual NYdb::NQuery::EStatsMode GetStatsMode() = 0;
5757
virtual TString ConvertPlan(const TString& plan) = 0;
5858
virtual TString GetPlanVisualization(const TString& plan) = 0;
59-
virtual TString GetQueryStat(const TString& plan, double& cpuUsage, TString* timeline) = 0;
59+
virtual TString GetQueryStat(const TString& plan, double& cpuUsage, TString* timeline, ui64 maxtimelineSize) = 0;
6060
virtual TPublicStat GetPublicStat(const TString& stat) = 0;
6161
virtual THashMap<TString, i64> GetFlatStat(TStringBuf plan) = 0;
6262
};
@@ -81,6 +81,7 @@ class PingTaskRequestBuilder {
8181
const TCompressor Compressor;
8282
std::unique_ptr<IPlanStatProcessor> Processor;
8383
bool ShowQueryTimeline = false;
84+
ui64 MaxQueryTimelineSize = 0;
8485
};
8586

8687
TString GetStatViewName(const ::NFq::TRunActorParams& params);

‎ydb/core/fq/libs/config/protos/common.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,5 @@ message TCommonConfig {
3030
bool UseNativeProtocolForClickHouse = 14;
3131
bool DisableSslForGenericDataSources = 15;
3232
bool ShowQueryTimeline = 16;
33+
uint64 MaxQueryTimelineSize = 17; // default: 200KB
3334
}

‎ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -594,6 +594,12 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeQue
594594
}
595595
}
596596
}
597+
598+
auto timeline = internal.timeline();
599+
if (timeline) {
600+
result.mutable_query()->mutable_timeline()->set_svg(timeline);
601+
}
602+
597603
if (!permissions.Check(TPermissions::VIEW_AST)) {
598604
result.mutable_query()->clear_ast();
599605
} else {

‎ydb/public/api/protos/draft/fq.proto

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,10 @@ message ResultSetMeta {
205205
bool truncated = 3;
206206
}
207207

208+
message QueryTimeline {
209+
string svg = 1; // No validation because generated on server side
210+
}
211+
208212
message Query {
209213
QueryMeta meta = 1;
210214
QueryContent content = 2;
@@ -214,6 +218,7 @@ message Query {
214218
QueryStatistics statistics = 6;
215219
repeated ResultSetMeta result_set_meta = 7;
216220
QueryAst ast = 8;
221+
QueryTimeline timeline = 9;
217222
}
218223

219224
message QueryStatistics {

‎ydb/tests/fq/plans/test_stats_mode.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ class TestStatsMode:
1414
@pytest.mark.parametrize(
1515
"stats_mode", ["STATS_MODE_NONE", "STATS_MODE_BASIC", "STATS_MODE_FULL", "STATS_MODE_PROFILE"]
1616
)
17-
def test_mode(self, kikimr, s3, client, stats_mode):
17+
def test_mode(self, kikimr, s3, client, stats_mode, yq_version):
1818
resource = boto3.resource(
1919
"s3", endpoint_url=s3.s3_url, aws_access_key_id="key", aws_secret_access_key="secret_key"
2020
)
@@ -50,6 +50,10 @@ def test_mode(self, kikimr, s3, client, stats_mode):
5050
query_id = client.create_query("simple", sql, type=fq.QueryContent.QueryType.ANALYTICS).result.query_id
5151
client.wait_query_status(query_id, fq.QueryMeta.COMPLETED)
5252

53+
if yq_version == "v2":
54+
result = client.describe_query(query_id).result
55+
assert "<svg" in result.query.timeline.svg
56+
5357
data = client.get_result_data(query_id)
5458
result_set = data.result.result_set
5559
logging.debug(str(result_set))

‎ydb/tests/tools/fq_runner/kikimr_utils.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ def apply_to_kikimr(self, request, kikimr):
112112
solomon_endpoint = os.environ.get('SOLOMON_URL')
113113
if solomon_endpoint is not None:
114114
kikimr.compute_plane.fq_config['common']['monitoring_endpoint'] = solomon_endpoint
115+
kikimr.control_plane.fq_config['common']['show_query_timeline'] = True
115116

116117

117118
class YQv2Extension(ExtensionPoint):

‎ydb/tests/tools/kqprun/src/kqp_runner.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -345,7 +345,7 @@ class TKqpRunner::TImpl {
345345

346346
try {
347347
double cpuUsage = 0.0;
348-
auto fullStat = StatProcessor_->GetQueryStat(convertedPlan, cpuUsage, nullptr);
348+
auto fullStat = StatProcessor_->GetQueryStat(convertedPlan, cpuUsage, nullptr, 0);
349349
auto flatStat = StatProcessor_->GetFlatStat(convertedPlan);
350350
auto publicStat = StatProcessor_->GetPublicStat(fullStat);
351351

0 commit comments

Comments
 (0)