Skip to content

Commit 6ff589b

Browse files
vitstnblinkov
vitstn
authored and
blinkov
committed
YQL-19594 Explicit lexers & parsers 2
commit_hash:2aa733486a55a04c39ff8bb1c7eaca3a85719647
1 parent 7500369 commit 6ff589b

File tree

31 files changed

+301
-41
lines changed

31 files changed

+301
-41
lines changed

‎yql/essentials/core/extract_predicate/ut/extract_predicate_ut.cpp

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@
77
#include <yql/essentials/sql/settings/translation_settings.h>
88
#include <yql/essentials/sql/sql.h>
99
#include <yql/essentials/sql/v1/sql.h>
10+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
11+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
12+
#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
13+
#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
1014
#include <yql/essentials/ast/yql_ast_annotation.h>
1115
#include <yql/essentials/ast/yql_expr.h>
1216
#include <yql/essentials/core/cbo/simple/cbo_simple.h>
@@ -54,9 +58,16 @@ Y_UNIT_TEST_SUITE(TYqlExtractPredicate) {
5458
NSQLTranslation::TTranslationSettings settings;
5559
settings.SyntaxVersion = 1;
5660

61+
NSQLTranslationV1::TLexers lexers;
62+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
63+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
64+
NSQLTranslationV1::TParsers parsers;
65+
parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
66+
parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
67+
5768
NSQLTranslation::TTranslators translators(
5869
nullptr,
59-
NSQLTranslationV1::MakeTranslator(),
70+
NSQLTranslationV1::MakeTranslator(lexers, parsers),
6071
nullptr
6172
);
6273

‎yql/essentials/core/extract_predicate/ut/ya.make

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ PEERDIR(
2323
yql/essentials/parser/pg_wrapper
2424
yql/essentials/sql
2525
yql/essentials/sql/v1
26+
yql/essentials/sql/v1/lexer/antlr4
27+
yql/essentials/sql/v1/lexer/antlr4_ansi
28+
yql/essentials/sql/v1/proto_parser/antlr4
29+
yql/essentials/sql/v1/proto_parser/antlr4_ansi
2630
)
2731

2832
YQL_LAST_ABI_VERSION()

‎yql/essentials/core/facade/ya.make

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,16 @@ PEERDIR(
2222
yql/essentials/core/qplayer/udf_resolver
2323
yql/essentials/sql
2424
yql/essentials/sql/v1
25+
yql/essentials/sql/v1/lexer/antlr4
26+
yql/essentials/sql/v1/lexer/antlr4_ansi
27+
yql/essentials/sql/v1/proto_parser/antlr4
28+
yql/essentials/sql/v1/proto_parser/antlr4_ansi
29+
#FIXME {
30+
yql/essentials/sql/v1/lexer/antlr3
31+
yql/essentials/sql/v1/lexer/antlr3_ansi
32+
yql/essentials/sql/v1/proto_parser/antlr3
33+
yql/essentials/sql/v1/proto_parser/antlr3_ansi
34+
#}
2535
yql/essentials/parser/pg_wrapper/interface
2636
yql/essentials/utils/log
2737
yql/essentials/core

‎yql/essentials/core/facade/yql_facade.cpp

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,16 @@
1212
#include <yql/essentials/core/services/yql_eval_params.h>
1313
#include <yql/essentials/sql/sql.h>
1414
#include <yql/essentials/sql/v1/sql.h>
15+
//FIXME {
16+
#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h>
17+
#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h>
18+
#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h>
19+
#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h>
20+
//}
21+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
22+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
23+
#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
24+
#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
1525
#include <yql/essentials/parser/pg_wrapper/interface/parser.h>
1626
#include <yql/essentials/utils/log/context.h>
1727
#include <yql/essentials/utils/log/profile.h>
@@ -731,9 +741,20 @@ bool TProgram::ParseSql(const NSQLTranslation::TTranslationSettings& settings)
731741
}
732742

733743
currentSettings->EmitReadsForExists = true;
744+
NSQLTranslationV1::TLexers lexers;
745+
lexers.Antlr3 = NSQLTranslationV1::MakeAntlr3LexerFactory();
746+
lexers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiLexerFactory();
747+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
748+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
749+
NSQLTranslationV1::TParsers parsers;
750+
parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory();
751+
parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory();
752+
parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
753+
parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
754+
734755
NSQLTranslation::TTranslators translators(
735756
nullptr,
736-
NSQLTranslationV1::MakeTranslator(),
757+
NSQLTranslationV1::MakeTranslator(lexers, parsers),
737758
NSQLTranslationPG::MakeTranslator()
738759
);
739760

‎yql/essentials/core/services/mounts/ya.make

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ PEERDIR(
1212
yql/essentials/utils/log
1313
yql/essentials/sql
1414
yql/essentials/sql/v1
15+
yql/essentials/sql/v1/lexer/antlr4
16+
yql/essentials/sql/v1/lexer/antlr4_ansi
17+
yql/essentials/sql/v1/proto_parser/antlr4
18+
yql/essentials/sql/v1/proto_parser/antlr4_ansi
1519
)
1620

1721
YQL_LAST_ABI_VERSION()

‎yql/essentials/core/services/mounts/yql_mounts.cpp

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@
33
#include <yql/essentials/core/yql_library_compiler.h>
44
#include <yql/essentials/sql/sql.h>
55
#include <yql/essentials/sql/v1/sql.h>
6+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
7+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
8+
#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
9+
#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
610
#include <yql/essentials/utils/log/profile.h>
711

812
#include <library/cpp/resource/resource.h>
@@ -126,9 +130,15 @@ namespace NYql {
126130
TUserDataTable mounts;
127131
LoadYqlDefaultMounts(mounts);
128132

133+
NSQLTranslationV1::TLexers lexers;
134+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
135+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
136+
NSQLTranslationV1::TParsers parsers;
137+
parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
138+
parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
129139
NSQLTranslation::TTranslators translators(
130140
nullptr,
131-
NSQLTranslationV1::MakeTranslator(),
141+
NSQLTranslationV1::MakeTranslator(lexers, parsers),
132142
nullptr
133143
);
134144

‎yql/essentials/core/ut/yql_library_compiler_ut.cpp

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@
22

33
#include "yql_library_compiler.h"
44

5-
#include <yql/essentials/sql/sql.h>
6-
#include <yql/essentials/sql/v1/sql.h>
7-
85
namespace NYql {
96

107
Y_UNIT_TEST_SUITE(TLibraryCompilerTests) {
@@ -14,7 +11,7 @@ Y_UNIT_TEST_SUITE(TLibraryCompilerTests) {
1411
static bool CompileAndLink(const THashMap<TString, TString>& libs, TExprContext& ctx) {
1512
NSQLTranslation::TTranslators translators(
1613
nullptr,
17-
NSQLTranslationV1::MakeTranslator(),
14+
nullptr,
1815
nullptr
1916
);
2017

@@ -36,7 +33,7 @@ Y_UNIT_TEST_SUITE(TLibraryCompilerTests) {
3633

3734
NSQLTranslation::TTranslators translators(
3835
nullptr,
39-
NSQLTranslationV1::MakeTranslator(),
36+
nullptr,
4037
nullptr
4138
);
4239

@@ -58,7 +55,7 @@ Y_UNIT_TEST_SUITE(TLibraryCompilerTests) {
5855

5956
NSQLTranslation::TTranslators translators(
6057
nullptr,
61-
NSQLTranslationV1::MakeTranslator(),
58+
nullptr,
6259
nullptr
6360
);
6461

‎yql/essentials/public/fastcheck/fastcheck.cpp

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,26 @@
77
#include <yql/essentials/core/yql_user_data_storage.h>
88
#include <yql/essentials/sql/sql.h>
99
#include <yql/essentials/sql/v1/sql.h>
10+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
11+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
12+
#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
13+
#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
1014
#include <yql/essentials/parser/pg_wrapper/interface/parser.h>
1115

1216
namespace NYql {
1317
namespace NFastCheck {
1418

1519
bool CheckProgram(const TString& program, const TOptions& options, TIssues& errors) {
20+
NSQLTranslationV1::TLexers lexers;
21+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
22+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
23+
NSQLTranslationV1::TParsers parsers;
24+
parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
25+
parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
26+
1627
NSQLTranslation::TTranslators translators(
1728
nullptr,
18-
NSQLTranslationV1::MakeTranslator(),
29+
NSQLTranslationV1::MakeTranslator(lexers, parsers),
1930
NSQLTranslationPG::MakeTranslator()
2031
);
2132

‎yql/essentials/public/fastcheck/translator.cpp

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
#include "check_runner.h"
22
#include <yql/essentials/sql/v1/sql.h>
3+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
4+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
5+
#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
6+
#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
37
#include <yql/essentials/sql/settings/translation_settings.h>
48
#include <yql/essentials/parser/pg_wrapper/interface/parser.h>
59

@@ -75,7 +79,14 @@ class TTranslatorRunner : public ICheckRunner {
7579
return res;
7680
}
7781

78-
auto astRes = NSQLTranslationV1::SqlToYql(request.Program, settings);
82+
NSQLTranslationV1::TLexers lexers;
83+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
84+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
85+
NSQLTranslationV1::TParsers parsers;
86+
parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
87+
parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
88+
89+
auto astRes = NSQLTranslationV1::SqlToYql(lexers, parsers, request.Program, settings);
7990
res.Success = astRes.IsOk();
8091
res.Issues = astRes.Issues;
8192
return res;

‎yql/essentials/public/purecalc/common/interface.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ TProgramFactoryOptions::TProgramFactoryOptions()
3333
, NativeYtTypeFlags(0)
3434
, UseSystemColumns(false)
3535
, UseWorkerPool(true)
36-
, UseAntlr4(false)
36+
, UseAntlr4(true)
3737
{
3838
}
3939

0 commit comments

Comments
 (0)