Skip to content

Commit 52daccf

Browse files
vitstnvitstn
committed
YQL-19594 Explicit lexers & parsers
commit_hash:6be543b7c5bff6ee474ee606c920197fb2569767
1 parent ca4b5f2 commit 52daccf

36 files changed

+358
-82
lines changed

‎yql/essentials/parser/lexer_common/ut/hints_ut.cpp

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#include <yql/essentials/parser/lexer_common/hints.h>
22
#include <yql/essentials/sql/v1/lexer/lexer.h>
3+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
34

45
#include <library/cpp/testing/unittest/registar.h>
56

@@ -8,9 +9,11 @@
89
using namespace NSQLTranslation;
910
using namespace NSQLTranslationV1;
1011

11-
TSQLHints CollectHints(const TString& query, bool antlr4Parser) {
12+
TSQLHints CollectHints(const TString& query) {
13+
NSQLTranslationV1::TLexers lexers;
14+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
1215
bool ansi = false;
13-
auto lexer = MakeLexer(ansi, antlr4Parser);
16+
auto lexer = MakeLexer(lexers, ansi, true);
1417
UNIT_ASSERT(lexer);
1518
TSQLHints result;
1619
NYql::TIssues issues;
@@ -27,20 +30,7 @@ TString SerializeHints(const TVector<TSQLHint>& hints) {
2730
Y_UNIT_TEST_SUITE(TLexerHintsTests) {
2831
Y_UNIT_TEST(Basic) {
2932
TString query = "/*+ some() */ SELECT /*+ foo(one) */ --+ bar(two)";
30-
auto hintsWithPos = CollectHints(query, false);
31-
UNIT_ASSERT(hintsWithPos.size() == 1);
32-
NYql::TPosition pos = hintsWithPos.begin()->first;
33-
TVector<TSQLHint> hints = hintsWithPos.begin()->second;
34-
35-
UNIT_ASSERT_EQUAL(pos.Row, 1);
36-
UNIT_ASSERT_EQUAL(pos.Column, 15);
37-
38-
TStringBuf expected = R"raw("foo":{"one"},"bar":{"two"})raw";
39-
UNIT_ASSERT_NO_DIFF(SerializeHints(hints), expected);
40-
}
41-
Y_UNIT_TEST(Antlr4) {
42-
TString query = "/*+ some() */ SELECT /*+ foo(one) */ --+ bar(two)";
43-
auto hintsWithPos = CollectHints(query, true);
33+
auto hintsWithPos = CollectHints(query);
4434
UNIT_ASSERT(hintsWithPos.size() == 1);
4535
NYql::TPosition pos = hintsWithPos.begin()->first;
4636
TVector<TSQLHint> hints = hintsWithPos.begin()->second;

‎yql/essentials/parser/lexer_common/ut/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ UNITTEST_FOR(yql/essentials/parser/lexer_common)
22

33
PEERDIR(
44
yql/essentials/sql/v1/lexer
5+
yql/essentials/sql/v1/lexer/antlr4
56
)
67

78

‎yql/essentials/public/fastcheck/format.cpp

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
#include "check_runner.h"
22
#include <yql/essentials/sql/v1/format/sql_format.h>
3+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
4+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
5+
#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
6+
#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
37
#include <util/string/builder.h>
48

59
namespace NYql {
@@ -53,7 +57,13 @@ class TFormatRunner : public ICheckRunner {
5357
settings.Antlr4Parser = true;
5458
settings.AnsiLexer = request.IsAnsiLexer;
5559

56-
auto formatter = NSQLFormat::MakeSqlFormatter(settings);
60+
NSQLTranslationV1::TLexers lexers;
61+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
62+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
63+
NSQLTranslationV1::TParsers parsers;
64+
parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
65+
parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
66+
auto formatter = NSQLFormat::MakeSqlFormatter(lexers, parsers, settings);
5767
TString formattedQuery;
5868
res.Success = formatter->Format(request.Program, formattedQuery, res.Issues);
5969
if (res.Success && formattedQuery != request.Program) {

‎yql/essentials/public/fastcheck/lexer.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#include "check_runner.h"
22
#include <yql/essentials/sql/v1/lexer/lexer.h>
3+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
4+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
35
#include <yql/essentials/sql/settings/translation_settings.h>
46

57
namespace NYql {
@@ -47,7 +49,10 @@ class TLexerRunner : public ICheckRunner {
4749
return res;
4850
}
4951

50-
auto lexer = NSQLTranslationV1::MakeLexer(settings.AnsiLexer, true);
52+
NSQLTranslationV1::TLexers lexers;
53+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
54+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
55+
auto lexer = NSQLTranslationV1::MakeLexer(lexers, settings.AnsiLexer, true);
5156
auto onNextToken = [&](NSQLTranslation::TParsedToken&& token) {
5257
Y_UNUSED(token);
5358
};

‎yql/essentials/public/fastcheck/parser.cpp

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
#include "check_runner.h"
22
#include <yql/essentials/sql/v1/lexer/lexer.h>
3+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
4+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
35
#include <yql/essentials/sql/v1/proto_parser/proto_parser.h>
6+
#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
7+
#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
48
#include <yql/essentials/sql/settings/translation_settings.h>
59
#include <yql/essentials/parser/pg_wrapper/interface/raw_parser.h>
610

@@ -73,15 +77,21 @@ class TParserRunner : public ICheckRunner {
7377
return res;
7478
}
7579

76-
auto lexer = NSQLTranslationV1::MakeLexer(settings.AnsiLexer, true);
80+
NSQLTranslationV1::TLexers lexers;
81+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
82+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
83+
auto lexer = NSQLTranslationV1::MakeLexer(lexers, settings.AnsiLexer, true);
7784
auto onNextToken = [&](NSQLTranslation::TParsedToken&& token) {
7885
Y_UNUSED(token);
7986
};
8087

8188
if (lexer->Tokenize(request.Program, request.File, onNextToken, res.Issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
8289
google::protobuf::Arena arena;
83-
auto msg = NSQLTranslationV1::SqlAST(request.Program, request.File, res.Issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS,
84-
settings.AnsiLexer, true, false, &arena);
90+
NSQLTranslationV1::TParsers parsers;
91+
parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
92+
parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
93+
auto msg = NSQLTranslationV1::SqlAST(parsers, request.Program, request.File, res.Issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS,
94+
settings.AnsiLexer, true, &arena);
8595
if (msg) {
8696
res.Success = true;
8797
}

‎yql/essentials/public/fastcheck/ya.make

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,11 @@ PEERDIR(
1919
yql/essentials/parser/pg_wrapper
2020
yql/essentials/providers/common/provider
2121
yql/essentials/sql/v1/lexer
22+
yql/essentials/sql/v1/lexer/antlr4
23+
yql/essentials/sql/v1/lexer/antlr4_ansi
2224
yql/essentials/sql/v1/proto_parser
25+
yql/essentials/sql/v1/proto_parser/antlr4
26+
yql/essentials/sql/v1/proto_parser/antlr4_ansi
2327
yql/essentials/sql/v1/format
2428
yql/essentials/sql/settings
2529
yql/essentials/parser/pg_wrapper/interface

‎yql/essentials/public/sql_format/sql_format.cpp

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#include "sql_format.h"
2+
3+
#include <yql/essentials/sql/v1/format/sql_format.h>
4+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
5+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
6+
#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
7+
#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
8+
9+
namespace NSQLFormat {
10+
11+
bool SqlFormatSimple(const TString& query, TString& formattedQuery, TString& error) {
12+
NSQLTranslationV1::TLexers lexers;
13+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
14+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
15+
NSQLTranslationV1::TParsers parsers;
16+
parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
17+
parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
18+
return SqlFormatSimple(lexers, parsers, query, formattedQuery, error);
19+
}
20+
21+
}

‎yql/essentials/public/sql_format/sql_format.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#pragma once
2+
#include <util/generic/string.h>
3+
4+
namespace NSQLFormat {
5+
6+
bool SqlFormatSimple(const TString& query, TString& formattedQuery, TString& error);
7+
8+
}

‎yql/essentials/public/sql_format/ya.make

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
LIBRARY()
2+
3+
SRCS(
4+
sql_format.cpp
5+
)
6+
7+
PEERDIR(
8+
yql/essentials/sql/v1/format
9+
yql/essentials/sql/v1/lexer/antlr4
10+
yql/essentials/sql/v1/lexer/antlr4_ansi
11+
yql/essentials/sql/v1/proto_parser/antlr4
12+
yql/essentials/sql/v1/proto_parser/antlr4_ansi
13+
)
14+
15+
END()

‎yql/essentials/public/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ RECURSE(
44
issue
55
purecalc
66
result_format
7+
sql_format
78
types
89
udf
910
)

0 commit comments

Comments
 (0)