Skip to content

Commit cd83d36

Browse files
vitstnvitstn
committed
Introduced two stages mode for type annotation, a FailMe callable to simulate optimization failures
В обычном select 1 \|\| 2 теперь выдается ошибка в InitialTypeAnnotation stage А ошибка на RepeatTypeAnnotation stage эскалируется до Severity=Fatal commit_hash:fb09693b4baf9e14bfe7dae9a8a9cd4b9e0e0142
1 parent d4510c4 commit cd83d36

File tree

41 files changed

+358
-61
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+358
-61
lines changed

‎yql/essentials/core/common_opt/yql_co_simple1.cpp

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6786,6 +6786,39 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
67866786
return node;
67876787
};
67886788

6789+
map["FailMe"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
6790+
YQL_CLOG(DEBUG, Core) << "Expand FailMe";
6791+
auto failureKind = node->Child(0)->Content();
6792+
if (failureKind == "expr") {
6793+
return ctx.Builder(node->Pos())
6794+
.Callable("String")
6795+
.Atom(0, "foo")
6796+
.Atom(1, "bar")
6797+
.Seal()
6798+
.Build();
6799+
}
6800+
6801+
if (failureKind == "type") {
6802+
return ctx.Builder(node->Pos())
6803+
.Callable("Int32")
6804+
.Atom(0, "1")
6805+
.Seal()
6806+
.Build();
6807+
}
6808+
6809+
if (failureKind == "constraint") {
6810+
return ctx.Builder(node->Pos())
6811+
.Callable("AsList")
6812+
.Callable(0, "String")
6813+
.Atom(0, "foo")
6814+
.Seal()
6815+
.Seal()
6816+
.Build();
6817+
}
6818+
6819+
throw yexception() << "Unknown failure kind: " << failureKind;
6820+
};
6821+
67896822
// will be applied to any callable after all above
67906823
map[""] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
67916824
YQL_ENSURE(node->IsCallable());

‎yql/essentials/core/facade/yql_facade.cpp

Lines changed: 52 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -874,7 +874,7 @@ TProgram::TFutureStatus TProgram::LineageAsync(const TString& username, IOutputS
874874
.AddPreTypeAnnotation()
875875
.AddExpressionEvaluation(*FunctionRegistry_)
876876
.AddIOAnnotation()
877-
.AddTypeAnnotation()
877+
.AddTypeAnnotation(TIssuesIds::CORE_TYPE_ANN, true)
878878
.AddPostTypeAnnotation()
879879
.Add(TExprOutputTransformer::Sync(ExprRoot_, traceOut), "ExprOutput")
880880
.AddCheckExecution(false)
@@ -937,7 +937,7 @@ TProgram::TFutureStatus TProgram::ValidateAsync(const TString& username, IOutput
937937
.AddPreTypeAnnotation()
938938
.AddExpressionEvaluation(*FunctionRegistry_)
939939
.AddIOAnnotation()
940-
.AddTypeAnnotation()
940+
.AddTypeAnnotation(TIssuesIds::CORE_TYPE_ANN, true)
941941
.Add(TExprOutputTransformer::Sync(ExprRoot_, exprOut, withTypes), "AstOutput")
942942
.Build();
943943

@@ -1012,7 +1012,7 @@ TProgram::TFutureStatus TProgram::OptimizeAsync(
10121012
.AddPreTypeAnnotation()
10131013
.AddExpressionEvaluation(*FunctionRegistry_)
10141014
.AddIOAnnotation()
1015-
.AddTypeAnnotation()
1015+
.AddTypeAnnotation(TIssuesIds::CORE_TYPE_ANN, true)
10161016
.AddPostTypeAnnotation()
10171017
.Add(TExprOutputTransformer::Sync(ExprRoot_, traceOut), "ExprOutput")
10181018
.AddOptimization()
@@ -1080,7 +1080,7 @@ TProgram::TFutureStatus TProgram::OptimizeAsyncWithConfig(
10801080
pipeline.AddPreTypeAnnotation();
10811081
pipeline.AddExpressionEvaluation(*FunctionRegistry_);
10821082
pipeline.AddIOAnnotation();
1083-
pipeline.AddTypeAnnotation();
1083+
pipeline.AddTypeAnnotation(TIssuesIds::CORE_TYPE_ANN, true);
10841084
pipeline.AddPostTypeAnnotation();
10851085
pipelineConf.AfterTypeAnnotation(&pipeline);
10861086

@@ -1139,7 +1139,7 @@ TProgram::TFutureStatus TProgram::LineageAsyncWithConfig(
11391139
pipeline.AddPreTypeAnnotation();
11401140
pipeline.AddExpressionEvaluation(*FunctionRegistry_);
11411141
pipeline.AddIOAnnotation();
1142-
pipeline.AddTypeAnnotation();
1142+
pipeline.AddTypeAnnotation(TIssuesIds::CORE_TYPE_ANN, true);
11431143
pipeline.AddPostTypeAnnotation();
11441144
pipelineConf.AfterTypeAnnotation(&pipeline);
11451145

@@ -1220,7 +1220,7 @@ TProgram::TFutureStatus TProgram::RunAsync(
12201220
pipeline.AddPreTypeAnnotation();
12211221
pipeline.AddExpressionEvaluation(*FunctionRegistry_);
12221222
pipeline.AddIOAnnotation();
1223-
pipeline.AddTypeAnnotation();
1223+
pipeline.AddTypeAnnotation(TIssuesIds::CORE_TYPE_ANN, true);
12241224
pipeline.AddPostTypeAnnotation();
12251225
pipeline.Add(TExprOutputTransformer::Sync(ExprRoot_, traceOut), "ExprOutput");
12261226
pipeline.AddOptimization();
@@ -1297,7 +1297,7 @@ TProgram::TFutureStatus TProgram::RunAsyncWithConfig(
12971297
pipeline.AddPreTypeAnnotation();
12981298
pipeline.AddExpressionEvaluation(*FunctionRegistry_);
12991299
pipeline.AddIOAnnotation();
1300-
pipeline.AddTypeAnnotation();
1300+
pipeline.AddTypeAnnotation(TIssuesIds::CORE_TYPE_ANN, true);
13011301
pipeline.AddPostTypeAnnotation();
13021302
pipelineConf.AfterTypeAnnotation(&pipeline);
13031303

@@ -1711,6 +1711,7 @@ TIssues TProgram::Issues() const {
17111711
result.AddIssues(ExprCtx_->IssueManager.GetIssues());
17121712
}
17131713
result.AddIssues(FinalIssues_);
1714+
CheckFatalIssues(result);
17141715
return result;
17151716
}
17161717

@@ -1720,9 +1721,53 @@ TIssues TProgram::CompletedIssues() const {
17201721
result.AddIssues(ExprCtx_->IssueManager.GetCompletedIssues());
17211722
}
17221723
result.AddIssues(FinalIssues_);
1724+
CheckFatalIssues(result);
17231725
return result;
17241726
}
17251727

1728+
void TProgram::CheckFatalIssues(TIssues& issues) const {
1729+
bool isFatal = false;
1730+
auto checkIssue = [&](const TIssue& issue) {
1731+
if (issue.GetSeverity() == TSeverityIds::S_FATAL) {
1732+
isFatal = true;
1733+
}
1734+
};
1735+
1736+
std::function<void(const TIssuePtr& issue)> recursiveCheck = [&](const TIssuePtr& issue) {
1737+
if (isFatal) {
1738+
return;
1739+
}
1740+
1741+
checkIssue(*issue);
1742+
for (const auto& subissue : issue->GetSubIssues()) {
1743+
recursiveCheck(subissue);
1744+
}
1745+
};
1746+
1747+
for (const auto& issue : issues) {
1748+
if (isFatal) {
1749+
break;
1750+
}
1751+
1752+
checkIssue(issue);
1753+
// check subissues
1754+
for (const auto& subissue : issue.GetSubIssues()) {
1755+
recursiveCheck(subissue);
1756+
}
1757+
}
1758+
1759+
if (isFatal) {
1760+
TIssue result;
1761+
result.SetMessage(
1762+
TStringBuilder()
1763+
<< "An abnormal situation found, so consider opening a bug report to YQL (st/YQLSUPPORT),"
1764+
<< " because more detailed information is only available in server side logs and/or "
1765+
<< "coredumps.");
1766+
result.SetCode(TIssuesIds::UNEXPECTED, TSeverityIds::S_FATAL);
1767+
issues.AddIssue(result);
1768+
}
1769+
}
1770+
17261771
TIssue MakeNoBlocksInfoIssue(const TVector<TString>& names, bool isTypes) {
17271772
TIssue result;
17281773
TString msg = TStringBuilder() << "Most frequent " << (isTypes ? "types " : "callables ")

‎yql/essentials/core/facade/yql_facade.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,7 @@ class TProgram: public TThrRefBase, private TNonCopyable
375375

376376
private:
377377
std::optional<bool> CheckFallbackIssues(const TIssues& issues);
378+
void CheckFatalIssues(TIssues& issues) const;
378379
void HandleSourceCode(TString& sourceCode);
379380
void HandleTranslationSettings(NSQLTranslation::TTranslationSettings& loadedSettings,
380381
const NSQLTranslation::TTranslationSettings*& currentSettings);

‎yql/essentials/core/issue/protos/issue_id.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ message TIssuesIds {
4646
// core errors
4747
CORE_GC_NODES_LIMIT_EXCEEDED = 1500;
4848
CORE_GC_STRINGS_LIMIT_EXCEEDED = 1501;
49+
CORE_RUNTIME_ERROR = 1502;
4950

5051
// kikimr
5152
KIKIMR_LOCKS_INVALIDATED = 2001;

‎yql/essentials/core/issue/yql_issue.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,10 @@ ids {
127127
code: CORE_GC_STRINGS_LIMIT_EXCEEDED
128128
severity: S_ERROR
129129
}
130+
ids {
131+
code: CORE_RUNTIME_ERROR
132+
severity: S_ERROR
133+
}
130134
ids {
131135
code: KIKIMR_LOCKS_INVALIDATED
132136
severity: S_ERROR

‎yql/essentials/core/services/yql_transform_pipeline.cpp

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,8 @@ TTransformationPipeline& TTransformationPipeline::AddIOAnnotation(bool withEpoch
106106
return *this;
107107
}
108108

109-
TTransformationPipeline& TTransformationPipeline::AddTypeAnnotation(EYqlIssueCode issueCode) {
110-
AddTypeAnnotationTransformer();
109+
TTransformationPipeline& TTransformationPipeline::AddTypeAnnotation(EYqlIssueCode issueCode, bool twoStages) {
110+
AddTypeAnnotationTransformer(issueCode, twoStages);
111111
Transformers_.push_back(TTransformStage(
112112
CreateFunctorTransformer(&CheckWholeProgramType),
113113
"CheckWholeProgramType", issueCode));
@@ -235,19 +235,42 @@ TTransformationPipeline& TTransformationPipeline::AddTableMetadataLoaderTransfor
235235
}
236236

237237
TTransformationPipeline& TTransformationPipeline::AddTypeAnnotationTransformer(
238-
TAutoPtr<IGraphTransformer> callableTransformer, EYqlIssueCode issueCode)
238+
TAutoPtr<IGraphTransformer> callableTransformer, EYqlIssueCode issueCode, ETypeCheckMode mode)
239239
{
240+
TString stageName;
241+
TString issue;
242+
switch (mode) {
243+
case ETypeCheckMode::Single:
244+
stageName = "TypeAnnotation";
245+
issue = "Type annotation";
246+
break;
247+
case ETypeCheckMode::Initial:
248+
stageName = "InitialTypeAnnotation";
249+
issue = "Type annotation";
250+
break;
251+
case ETypeCheckMode::Repeat:
252+
stageName = "RepeatTypeAnnotation";
253+
issue = "Type annotation (repeat)";
254+
break;
255+
}
256+
240257
Transformers_.push_back(TTransformStage(
241-
CreateTypeAnnotationTransformer(callableTransformer, *TypeAnnotationContext_),
242-
"TypeAnnotation",
243-
issueCode));
258+
CreateTypeAnnotationTransformer(callableTransformer, *TypeAnnotationContext_, mode), stageName, issueCode, issue));
244259
return *this;
245260
}
246261

247-
TTransformationPipeline& TTransformationPipeline::AddTypeAnnotationTransformer(EYqlIssueCode issueCode)
262+
TTransformationPipeline& TTransformationPipeline::AddTypeAnnotationTransformer(EYqlIssueCode issueCode, bool twoStages)
248263
{
249-
auto callableTransformer = CreateExtCallableTypeAnnotationTransformer(*TypeAnnotationContext_);
250-
return AddTypeAnnotationTransformer(callableTransformer, issueCode);
264+
if (twoStages) {
265+
std::shared_ptr<IGraphTransformer> callableTransformer(CreateExtCallableTypeAnnotationTransformer(*TypeAnnotationContext_).Release());
266+
AddTypeAnnotationTransformer(MakeSharedTransformerProxy(callableTransformer), issueCode, ETypeCheckMode::Initial);
267+
AddTypeAnnotationTransformer(MakeSharedTransformerProxy(callableTransformer), issueCode, ETypeCheckMode::Repeat);
268+
} else {
269+
auto callableTransformer = CreateExtCallableTypeAnnotationTransformer(*TypeAnnotationContext_);
270+
AddTypeAnnotationTransformer(callableTransformer, issueCode, ETypeCheckMode::Single);
271+
}
272+
273+
return *this;
251274
}
252275

253276
TAutoPtr<IGraphTransformer> TTransformationPipeline::Build(bool useIssueScopes) {

‎yql/essentials/core/services/yql_transform_pipeline.h

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <yql/essentials/core/yql_type_annotation.h>
55
#include <yql/essentials/core/yql_execution.h>
66
#include <yql/essentials/core/issue/yql_issue.h>
7+
#include <yql/essentials/core/type_ann/type_ann_expr.h>
78

89
#include <util/generic/ptr.h>
910

@@ -29,7 +30,7 @@ class TTransformationPipeline
2930
IGraphTransformer* calcTransfomer = nullptr, EYqlIssueCode issueCode = TIssuesIds::CORE_EXPR_EVALUATION);
3031
TTransformationPipeline& AddPreIOAnnotation(bool withEpochsTransformer = true, EYqlIssueCode issueCode = TIssuesIds::CORE_PRE_TYPE_ANN);
3132
TTransformationPipeline& AddIOAnnotation(bool withEpochsTransformer = true, EYqlIssueCode issueCode = TIssuesIds::CORE_PRE_TYPE_ANN);
32-
TTransformationPipeline& AddTypeAnnotation(EYqlIssueCode issueCode = TIssuesIds::CORE_TYPE_ANN);
33+
TTransformationPipeline& AddTypeAnnotation(EYqlIssueCode issueCode = TIssuesIds::CORE_TYPE_ANN, bool twoStages = false);
3334
TTransformationPipeline& AddPostTypeAnnotation(bool forSubGraph = false, bool disableConstraintCheck = false, EYqlIssueCode issueCode = TIssuesIds::CORE_POST_TYPE_ANN);
3435
TTransformationPipeline& AddCommonOptimization(EYqlIssueCode issueCode = TIssuesIds::CORE_OPTIMIZATION);
3536
TTransformationPipeline& AddFinalCommonOptimization(EYqlIssueCode issueCode = TIssuesIds::CORE_OPTIMIZATION);
@@ -40,8 +41,9 @@ class TTransformationPipeline
4041

4142
TTransformationPipeline& AddIntentDeterminationTransformer(EYqlIssueCode issueCode = TIssuesIds::CORE_INTENT);
4243
TTransformationPipeline& AddTableMetadataLoaderTransformer(EYqlIssueCode issueCode = TIssuesIds::CORE_TABLE_METADATA_LOADER);
43-
TTransformationPipeline& AddTypeAnnotationTransformer(TAutoPtr<IGraphTransformer> callableTransformer, EYqlIssueCode issueCode = TIssuesIds::CORE_TYPE_ANN);
44-
TTransformationPipeline& AddTypeAnnotationTransformer(EYqlIssueCode issueCode = TIssuesIds::CORE_TYPE_ANN);
44+
TTransformationPipeline& AddTypeAnnotationTransformer(TAutoPtr<IGraphTransformer> callableTransformer, EYqlIssueCode issueCode = TIssuesIds::CORE_TYPE_ANN,
45+
ETypeCheckMode mode = ETypeCheckMode::Single);
46+
TTransformationPipeline& AddTypeAnnotationTransformer(EYqlIssueCode issueCode = TIssuesIds::CORE_TYPE_ANN, bool twoStages = false);
4547

4648
TTransformationPipeline& Add(TAutoPtr<IGraphTransformer> transformer, const TString& stageName,
4749
EYqlIssueCode issueCode = TIssuesIds::DEFAULT_ERROR, const TString& issueMessage = {});

‎yql/essentials/core/type_ann/type_ann_core.cpp

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -579,6 +579,30 @@ namespace NTypeAnnImpl {
579579
return IGraphTransformer::TStatus::Ok;
580580
}
581581

582+
IGraphTransformer::TStatus FailMeWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
583+
if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
584+
return IGraphTransformer::TStatus::Error;
585+
}
586+
587+
if (!EnsureAtom(*input->Child(0), ctx.Expr)) {
588+
return IGraphTransformer::TStatus::Error;
589+
}
590+
591+
auto failureKind = input->Child(0)->Content();
592+
if (failureKind == "expr") {
593+
input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(NUdf::EDataSlot::String));
594+
} else if (failureKind == "type") {
595+
input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(NUdf::EDataSlot::String));
596+
} else if (failureKind == "constraint") {
597+
input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TDataExprType>(NUdf::EDataSlot::String)));
598+
} else {
599+
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Unknown failure kind: " << failureKind));
600+
return IGraphTransformer::TStatus::Error;
601+
}
602+
603+
return IGraphTransformer::TStatus::Ok;
604+
}
605+
582606
IGraphTransformer::TStatus DataWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
583607
if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
584608
return IGraphTransformer::TStatus::Error;
@@ -12197,6 +12221,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
1219712221
}
1219812222

1219912223
TSyncFunctionsMap::TSyncFunctionsMap() {
12224+
Functions["FailMe"] = &FailMeWrapper;
1220012225
Functions["Data"] = &DataWrapper;
1220112226
Functions["DataOrOptionalData"] = &DataWrapper;
1220212227
Functions["DataSource"] = &DataSourceWrapper;

‎yql/essentials/core/type_ann/type_ann_core.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ IGraphTransformer::TStatus ValidateDataSink(const TExprNode::TPtr& node, TExprCo
1212
IGraphTransformer::TStatus ValidateProviders(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExprContext& ctx, const TTypeAnnotationContext& types);
1313

1414
TAutoPtr<IGraphTransformer> CreateIntentDeterminationTransformer(const TTypeAnnotationContext& types);
15-
TAutoPtr<IGraphTransformer> CreateExtCallableTypeAnnotationTransformer(TTypeAnnotationContext& types, bool instantOnly = false);
15+
16+
TAutoPtr<IGraphTransformer> CreateExtCallableTypeAnnotationTransformer(TTypeAnnotationContext& types,
17+
bool instantOnly = false);
1618

1719
const THashSet<TString>& GetBuiltinFunctions();
1820

0 commit comments

Comments
 (0)