Skip to content

Commit e479202

Browse files
vitstnvitstn
committed
YQL-19845 support of lang version checking inside facade
commit_hash:5cfb2a0aa2904106df4ae69b9311bcc5a695928d
1 parent 345eb06 commit e479202

File tree

15 files changed

+201
-1
lines changed

15 files changed

+201
-1
lines changed

‎yql/essentials/core/facade/ya.make

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ PEERDIR(
3030
yql/essentials/utils/log
3131
yql/essentials/core
3232
yql/essentials/core/type_ann
33+
yql/essentials/core/langver
34+
yql/essentials/public/langver
3335
yql/essentials/providers/common/config
3436
yql/essentials/providers/common/proto
3537
yql/essentials/providers/common/provider

‎yql/essentials/core/facade/yql_facade.cpp

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include <yql/essentials/core/type_ann/type_ann_expr.h>
1111
#include <yql/essentials/core/services/yql_plan.h>
1212
#include <yql/essentials/core/services/yql_eval_params.h>
13+
#include <yql/essentials/core/langver/yql_core_langver.h>
1314
#include <yql/essentials/sql/sql.h>
1415
#include <yql/essentials/sql/v1/sql.h>
1516
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
@@ -39,6 +40,7 @@
3940

4041
#include <util/stream/file.h>
4142
#include <util/stream/null.h>
43+
#include <util/string/cast.h>
4244
#include <util/string/join.h>
4345
#include <util/string/split.h>
4446
#include <util/generic/guid.h>
@@ -56,6 +58,7 @@ const size_t DEFAULT_AST_BUF_SIZE = 1024;
5658
const size_t DEFAULT_PLAN_BUF_SIZE = 1024;
5759
const TString FacadeComponent = "Facade";
5860
const TString SourceCodeLabel = "SourceCode";
61+
const TString LangVerLabel = "LangVer";
5962
const TString GatewaysLabel = "Gateways";
6063
const TString ParametersLabel = "Parameters";
6164
const TString TranslationLabel = "Translation";
@@ -166,6 +169,14 @@ void TProgramFactory::EnableRangeComputeFor() {
166169
EnableRangeComputeFor_ = true;
167170
}
168171

172+
void TProgramFactory::SetLanguageVersion(TLangVersion version) {
173+
LangVer_ = version;
174+
}
175+
176+
void TProgramFactory::SetMaxLanguageVersion(TLangVersion version) {
177+
MaxLangVer_ = version;
178+
}
179+
169180
void TProgramFactory::AddUserDataTable(const TUserDataTable& userDataTable) {
170181
for (const auto& p : userDataTable) {
171182
if (!UserDataTable_.emplace(p).second) {
@@ -242,7 +253,7 @@ TProgramPtr TProgramFactory::Create(
242253

243254
// make UserDataTable_ copy here
244255
return new TProgram(FunctionRegistry_, randomProvider, timeProvider, NextUniqueId_, DataProvidersInit_,
245-
UserDataTable_, Credentials_, moduleResolver, urlListerManager,
256+
LangVer_, MaxLangVer_, UserDataTable_, Credentials_, moduleResolver, urlListerManager,
246257
udfResolver, udfIndex, udfIndexPackageSet, FileStorage_, UrlPreprocessing_,
247258
GatewaysConfig_, filename, sourceCode, sessionId, Runner_, EnableRangeComputeFor_, ArrowResolver_, hiddenMode,
248259
qContext, gatewaysForMerge);
@@ -257,6 +268,8 @@ TProgram::TProgram(
257268
const TIntrusivePtr<ITimeProvider> timeProvider,
258269
ui64 nextUniqueId,
259270
const TVector<TDataProviderInitializer>& dataProvidersInit,
271+
TLangVersion langVer,
272+
TLangVersion maxLangVer,
260273
const TUserDataTable& userDataTable,
261274
const TCredentials::TPtr& credentials,
262275
const IModuleResolver::TPtr& modules,
@@ -284,6 +297,8 @@ TProgram::TProgram(
284297
, AstRoot_(nullptr)
285298
, Modules_(modules)
286299
, DataProvidersInit_(dataProvidersInit)
300+
, LangVer_(langVer)
301+
, MaxLangVer_(maxLangVer)
287302
, Credentials_(MakeIntrusive<NYql::TCredentials>(*credentials))
288303
, UrlListerManager_(urlListerManager)
289304
, UdfResolver_(udfResolver)
@@ -587,6 +602,14 @@ TString TProgram::GetSessionId() const {
587602
}
588603
}
589604

605+
void TProgram::SetLanguageVersion(TLangVersion version) {
606+
LangVer_ = version;
607+
}
608+
609+
void TProgram::SetMaxLanguageVersion(TLangVersion version) {
610+
MaxLangVer_ = version;
611+
}
612+
590613
void TProgram::AddCredentials(const TVector<std::pair<TString, TCredential>>& credentials) {
591614
Y_ENSURE(!TypeCtx_, "TypeCtx_ already created");
592615

@@ -740,8 +763,41 @@ bool TProgram::CheckParameters() {
740763
return true;
741764
}
742765

766+
bool TProgram::ValidateLangVersion() {
767+
if (QContext_.CanRead()) {
768+
auto loaded = QContext_.GetReader()->Get({FacadeComponent, LangVerLabel}).GetValueSync();
769+
if (loaded.Defined()) {
770+
LangVer_ = FromString<TLangVersion>(loaded->Value);
771+
} else {
772+
LangVer_ = UnknownLangVersion;
773+
}
774+
775+
return true;
776+
}
777+
778+
if (QContext_.CanWrite()) {
779+
QContext_.GetWriter()->Put({FacadeComponent, LangVerLabel}, ToString(LangVer_)).GetValueSync();
780+
}
781+
782+
TMaybe<TIssue> issue;
783+
auto ret = CheckLangVersion(LangVer_, MaxLangVer_, issue);
784+
if (issue) {
785+
if (!ExprCtx_) {
786+
ExprCtx_.Reset(new TExprContext(NextUniqueId_));
787+
}
788+
789+
ExprCtx_->AddError(*issue);
790+
}
791+
792+
return ret;
793+
}
794+
743795
bool TProgram::ParseYql() {
744796
YQL_PROFILE_FUNC(TRACE);
797+
if (!ValidateLangVersion()) {
798+
return false;
799+
}
800+
745801
if (!CheckParameters()) {
746802
return false;
747803
}
@@ -769,6 +825,10 @@ bool TProgram::ParseSql() {
769825
bool TProgram::ParseSql(const NSQLTranslation::TTranslationSettings& settings)
770826
{
771827
YQL_PROFILE_FUNC(TRACE);
828+
if (!ValidateLangVersion()) {
829+
return false;
830+
}
831+
772832
if (!CheckParameters()) {
773833
return false;
774834
}
@@ -788,6 +848,8 @@ bool TProgram::ParseSql(const NSQLTranslation::TTranslationSettings& settings)
788848
}
789849

790850
currentSettings->EmitReadsForExists = true;
851+
currentSettings->LangVer = LangVer_;
852+
791853
NSQLTranslationV1::TLexers lexers;
792854
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
793855
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
@@ -1912,6 +1974,7 @@ TString TProgram::ResultsAsString() const {
19121974
TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& username) {
19131975
auto typeAnnotationContext = MakeIntrusive<TTypeAnnotationContext>();
19141976

1977+
typeAnnotationContext->LangVer = LangVer_;
19151978
typeAnnotationContext->UserDataStorage = UserDataStorage_;
19161979
typeAnnotationContext->Credentials = Credentials_;
19171980
typeAnnotationContext->Modules = Modules_;

‎yql/essentials/core/facade/yql_facade.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include <yql/essentials/providers/result/provider/yql_result_provider.h>
1414
#include <yql/essentials/providers/common/proto/gateways_config.pb.h>
1515
#include <yql/essentials/public/issue/yql_issue.h>
16+
#include <yql/essentials/public/langver/yql_langver.h>
1617
#include <yql/essentials/sql/sql.h>
1718

1819
#include <library/cpp/random_provider/random_provider.h>
@@ -50,6 +51,8 @@ class TProgramFactory: public TThrRefBase, private TMoveOnly
5051
const TVector<TDataProviderInitializer>& dataProvidersInit,
5152
const TString& runner);
5253

54+
void SetLanguageVersion(TLangVersion version);
55+
void SetMaxLanguageVersion(TLangVersion version);
5356
void AddUserDataTable(const TUserDataTable& userDataTable);
5457
void SetCredentials(TCredentials::TPtr credentials);
5558
void SetGatewaysConfig(const TGatewaysConfig* gatewaysConfig);
@@ -83,6 +86,8 @@ class TProgramFactory: public TThrRefBase, private TMoveOnly
8386
const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry_;
8487
const ui64 NextUniqueId_;
8588
TVector<TDataProviderInitializer> DataProvidersInit_;
89+
TLangVersion LangVer_ = UnknownLangVersion;
90+
TLangVersion MaxLangVer_ = UnknownLangVersion;
8691
TUserDataTable UserDataTable_;
8792
TCredentials::TPtr Credentials_;
8893
const TGatewaysConfig* GatewaysConfig_;
@@ -111,6 +116,9 @@ class TProgram: public TThrRefBase, private TNonCopyable
111116
public:
112117
~TProgram();
113118

119+
void SetLanguageVersion(TLangVersion version);
120+
void SetMaxLanguageVersion(TLangVersion version);
121+
114122
void AddCredentials(const TVector<std::pair<TString, TCredential>>& credentials);
115123
void ClearCredentials();
116124

@@ -334,6 +342,8 @@ class TProgram: public TThrRefBase, private TNonCopyable
334342
const TIntrusivePtr<ITimeProvider> timeProvider,
335343
ui64 nextUniqueId,
336344
const TVector<TDataProviderInitializer>& dataProvidersInit,
345+
TLangVersion langVer,
346+
TLangVersion maxLangVer,
337347
const TUserDataTable& userDataTable,
338348
const TCredentials::TPtr& credentials,
339349
const IModuleResolver::TPtr& modules,
@@ -359,6 +369,7 @@ class TProgram: public TThrRefBase, private TNonCopyable
359369
TTypeAnnotationContextPtr ProvideAnnotationContext(const TString& username);
360370
bool CollectUsedClusters();
361371
bool CheckParameters();
372+
bool ValidateLangVersion();
362373

363374
NThreading::TFuture<void> OpenSession(const TString& username);
364375

@@ -395,6 +406,8 @@ class TProgram: public TThrRefBase, private TNonCopyable
395406
const IModuleResolver::TPtr Modules_;
396407

397408
TVector<TDataProviderInitializer> DataProvidersInit_;
409+
TLangVersion LangVer_;
410+
TLangVersion MaxLangVer_;
398411
TAdaptiveLock DataProvidersLock_;
399412
TVector<TDataProviderInfo> DataProviders_;
400413
TYqlOperationOptions OperationOptions_;

‎yql/essentials/core/issue/protos/issue_id.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ message TIssuesIds {
3838
CORE_LEGACY_REGEX_ENGINE = 1110;
3939
CORE_ALIAS_SHADOWS_COLUMN = 1111;
4040
CORE_LINEAGE_INTERNAL_ERROR = 1112;
41+
CORE_DEPRECATED_LANG_VER = 1113;
42+
CORE_UNSUPPORTED_LANG_VER = 1114;
4143

4244
// core informational
4345
CORE_TOP_UNSUPPORTED_BLOCK_TYPES = 1200;

‎yql/essentials/core/issue/yql_issue.txt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -675,6 +675,14 @@ ids {
675675
code: CORE_LINEAGE_INTERNAL_ERROR
676676
severity: S_WARNING
677677
}
678+
ids {
679+
code: CORE_DEPRECATED_LANG_VER
680+
severity: S_WARNING
681+
}
682+
ids {
683+
code: CORE_UNSUPPORTED_LANG_VER
684+
severity: S_WARNING
685+
}
678686
ids {
679687
code: YT_SECURE_DATA_IN_COMMON_TMP
680688
severity: S_WARNING

‎yql/essentials/core/langver/ya.make

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
LIBRARY()
2+
3+
SRCS(
4+
yql_core_langver.cpp
5+
)
6+
7+
PEERDIR(
8+
yql/essentials/core/issue
9+
)
10+
11+
END()

‎yql/essentials/core/langver/yql_core_langver.cpp

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#include "yql_core_langver.h"
2+
3+
#include <yql/essentials/core/issue/yql_issue.h>
4+
#include <util/string/builder.h>
5+
6+
namespace NYql {
7+
8+
namespace {
9+
10+
void WriteVersion(TStringBuilder& builder, TLangVersion version) {
11+
TLangVersionBuffer buffer;
12+
TStringBuf str;
13+
if (!FormatLangVersion(version, buffer, str)) {
14+
builder << "unknown";
15+
} else {
16+
builder << str;
17+
}
18+
}
19+
20+
}
21+
22+
bool CheckLangVersion(TLangVersion ver, TLangVersion max, TMaybe<TIssue>& issue) {
23+
if (ver != UnknownLangVersion && !IsValidLangVersion(ver)) {
24+
TStringBuilder builder;
25+
builder << "Invalid YQL language version: '";
26+
WriteVersion(builder, ver);
27+
builder << "'";
28+
issue = TIssue({}, builder);
29+
return false;
30+
}
31+
32+
if (!IsAvalableLangVersion(ver, max)) {
33+
TStringBuilder builder;
34+
builder << "YQL language version '";
35+
WriteVersion(builder, ver);
36+
builder << "' is not available, maximum version is '";
37+
WriteVersion(builder, max);
38+
builder << "'";
39+
issue = TIssue({}, builder);
40+
return false;
41+
}
42+
43+
if (IsDeprecatedLangVersion(ver, max)) {
44+
TStringBuilder builder;
45+
builder << "YQL language version '";
46+
WriteVersion(builder, ver);
47+
builder << "' is deprecated, consider to upgrade";
48+
issue = YqlIssue({}, EYqlIssueCode::TIssuesIds_EIssueCode_CORE_DEPRECATED_LANG_VER, builder);
49+
return true;
50+
}
51+
52+
if (IsUnsupportedLangVersion(ver, max)) {
53+
TStringBuilder builder;
54+
builder << "YQL language version '";
55+
WriteVersion(builder, ver);
56+
builder << "' is unsupported, consider to upgrade. Queries may fail";
57+
issue = YqlIssue({}, EYqlIssueCode::TIssuesIds_EIssueCode_CORE_UNSUPPORTED_LANG_VER, builder);
58+
return true;
59+
}
60+
61+
return true;
62+
}
63+
64+
} // namespace NYql

‎yql/essentials/core/langver/yql_core_langver.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#pragma once
2+
#include <yql/essentials/public/issue/yql_issue.h>
3+
#include <yql/essentials/public/langver/yql_langver.h>
4+
5+
namespace NYql {
6+
7+
bool CheckLangVersion(TLangVersion ver, TLangVersion max, TMaybe<TIssue>& issue);
8+
9+
} // namespace NYql

‎yql/essentials/core/ya.make

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ PEERDIR(
7575
yql/essentials/protos
7676
yql/essentials/public/udf
7777
yql/essentials/public/udf/tz
78+
yql/essentials/public/langver
7879
yql/essentials/sql/settings
7980
yql/essentials/sql
8081
yql/essentials/utils
@@ -103,6 +104,7 @@ RECURSE(
103104
dqs_expr_nodes
104105
file_storage
105106
issue
107+
langver
106108
minsketch
107109
pg_ext
108110
pg_settings

‎yql/essentials/core/yql_type_annotation.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
#include <yql/essentials/public/udf/udf_validate.h>
1111
#include <yql/essentials/public/udf/udf_log.h>
12+
#include <yql/essentials/public/langver/yql_langver.h>
1213
#include <yql/essentials/core/credentials/yql_credentials.h>
1314
#include <yql/essentials/core/url_lister/interface/url_lister_manager.h>
1415
#include <yql/essentials/core/qplayer/storage/interface/yql_qstorage.h>
@@ -375,6 +376,7 @@ inline TString GetRandomKey<TGUID>() {
375376
}
376377

377378
struct TTypeAnnotationContext: public TThrRefBase {
379+
TLangVersion LangVer = UnknownLangVersion;
378380
THashMap<TString, TIntrusivePtr<TOptimizerStatistics::TColumnStatMap>> ColumnStatisticsByTableName;
379381
THashMap<ui64, std::shared_ptr<TOptimizerStatistics>> StatisticsMap;
380382
TIntrusivePtr<ITimeProvider> TimeProvider;

0 commit comments

Comments
 (0)