Skip to content

Commit 11433e1

Browse files
OrlovPavelroot
OrlovPavel
and
root
authored
Migrating from antlr3 to antlr4 (#2977)
Co-authored-by: root <p.g.orlov@tinkoff.ru>
1 parent 085cd3d commit 11433e1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+14126
-1773
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ __pycache__/
4343
.idea/
4444
.vscode/
4545
.clangd
46+
.antlr/
4647

4748
# KDevelop IDE
4849
*.kdev4

‎ydb/library/yql/parser/lexer_common/ut/hints_ut.cpp

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@
88
using namespace NSQLTranslation;
99
using namespace NSQLTranslationV1;
1010

11-
TSQLHints CollectHints(const TString& query) {
11+
TSQLHints CollectHints(const TString& query, bool antlr4Parser) {
1212
bool ansi = false;
13-
auto lexer = MakeLexer(ansi);
13+
auto lexer = MakeLexer(ansi, antlr4Parser);
1414
UNIT_ASSERT(lexer);
1515
TSQLHints result;
1616
NYql::TIssues issues;
@@ -27,7 +27,20 @@ TString SerializeHints(const TVector<TSQLHint>& hints) {
2727
Y_UNIT_TEST_SUITE(TLexerHintsTests) {
2828
Y_UNIT_TEST(Basic) {
2929
TString query = "/*+ some() */ SELECT /*+ foo(one) */ --+ bar(two)";
30-
auto hintsWithPos = CollectHints(query);
30+
auto hintsWithPos = CollectHints(query, false);
31+
UNIT_ASSERT(hintsWithPos.size() == 1);
32+
NYql::TPosition pos = hintsWithPos.begin()->first;
33+
TVector<TSQLHint> hints = hintsWithPos.begin()->second;
34+
35+
UNIT_ASSERT_EQUAL(pos.Row, 1);
36+
UNIT_ASSERT_EQUAL(pos.Column, 15);
37+
38+
TStringBuf expected = R"raw("foo":{"one"},"bar":{"two"})raw";
39+
UNIT_ASSERT_NO_DIFF(SerializeHints(hints), expected);
40+
}
41+
Y_UNIT_TEST(Antlr4) {
42+
TString query = "/*+ some() */ SELECT /*+ foo(one) */ --+ bar(two)";
43+
auto hintsWithPos = CollectHints(query, true);
3144
UNIT_ASSERT(hintsWithPos.size() == 1);
3245
NYql::TPosition pos = hintsWithPos.begin()->first;
3346
TVector<TSQLHint> hints = hintsWithPos.begin()->second;

‎ydb/library/yql/parser/proto_ast/gen/jsonpath/ya.make

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ IF (CPP_PROTO)
1010
SET(PROTOBUF_SUFFIX_PATH .pb.h)
1111
SET(LEXER_PARSER_NAMESPACE NALP)
1212

13-
1413
CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/Cpp/Cpp.stg.in ${antlr_templates}/Cpp/Cpp.stg)
1514
CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/protobuf/protobuf.stg.in ${antlr_templates}/protobuf/protobuf.stg)
1615

@@ -28,6 +27,10 @@ IF (CPP_PROTO)
2827

2928
NO_COMPILER_WARNINGS()
3029

30+
ADDINCL(
31+
GLOBAL contrib/libs/antlr4_cpp_runtime/src
32+
)
33+
3134
INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/ya.make.incl)
3235

3336
RUN_ANTLR(

‎ydb/library/yql/parser/proto_ast/gen/v0/ya.make

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ NO_COMPILER_WARNINGS()
2020

2121
INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/ya.make.incl)
2222

23+
ADDINCL(
24+
GLOBAL contrib/libs/antlr4_cpp_runtime/src
25+
)
26+
2327
RUN_ANTLR(
2428
${sql_grammar}
2529
-lib .

‎ydb/library/yql/parser/proto_ast/gen/v1/ya.make

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@ ENDIF()
2828

2929
NO_COMPILER_WARNINGS()
3030

31+
ADDINCL(
32+
GLOBAL contrib/libs/antlr4_cpp_runtime/src
33+
)
34+
3135
INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/ya.make.incl)
3236

3337
RUN_ANTLR(

‎ydb/library/yql/parser/proto_ast/gen/v1_ansi/ya.make

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@ ENDIF()
2828

2929
NO_COMPILER_WARNINGS()
3030

31+
ADDINCL(
32+
GLOBAL contrib/libs/antlr4_cpp_runtime/src
33+
)
34+
3135
INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/ya.make.incl)
3236

3337
RUN_ANTLR(

‎ydb/library/yql/parser/proto_ast/gen/v1_ansi_antlr4/epilogue.cmake

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
set(GRAMMAR_STRING_CORE_SINGLE "~([']) | (QUOTE_SINGLE QUOTE_SINGLE)")
2+
set(GRAMMAR_STRING_CORE_DOUBLE "~([\"]) | (QUOTE_DOUBLE QUOTE_DOUBLE)")
3+
set(GRAMMAR_MULTILINE_COMMENT_CORE "MULTILINE_COMMENT | .")
4+
5+
configure_file(
6+
${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/SQLv1Antlr4.g.in
7+
${CMAKE_BINARY_DIR}/ydb/library/yql/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4.g
8+
)
9+
10+

‎ydb/library/yql/parser/proto_ast/gen/v1_ansi_antlr4/ya.make

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
LIBRARY()
2+
3+
PEERDIR (
4+
ydb/library/yql/parser/proto_ast/gen/v1_proto_split
5+
)
6+
7+
SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR})
8+
SET(antlr_templates ${antlr_output}/org/antlr/v4/tool/templates/codegen)
9+
SET(sql_grammar ${antlr_output}/SQLv1Antlr4.g)
10+
11+
SET(ANTLR_PACKAGE_NAME NSQLv1Generated)
12+
SET(PROTOBUF_HEADER_PATH ydb/library/yql/parser/proto_ast/gen/v1_proto_split)
13+
SET(PROTOBUF_SUFFIX_PATH .pb.main.h)
14+
15+
SET(LEXER_PARSER_NAMESPACE NALPAnsiAntlr4)
16+
17+
SET(GRAMMAR_STRING_CORE_SINGLE "\"~([']) | (QUOTE_SINGLE QUOTE_SINGLE)\"")
18+
SET(GRAMMAR_STRING_CORE_DOUBLE "\"~([#DOUBLE_QUOTE#]) | (QUOTE_DOUBLE QUOTE_DOUBLE)\"")
19+
SET(GRAMMAR_MULTILINE_COMMENT_CORE "\"MULTILINE_COMMENT | .\"")
20+
21+
CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg.in ${antlr_templates}/Cpp/Cpp.stg)
22+
CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg.in ${antlr_templates}/Cpp/Files.stg)
23+
24+
IF(EXPORT_CMAKE)
25+
MANUAL_GENERATION(${sql_grammar})
26+
ELSE()
27+
CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/sql/v1/SQLv1Antlr4.g.in ${sql_grammar})
28+
ENDIF()
29+
30+
NO_COMPILER_WARNINGS()
31+
32+
ADDINCL(
33+
GLOBAL contrib/libs/antlr4_cpp_runtime/src
34+
)
35+
36+
INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/ya.make.incl)
37+
38+
RUN_ANTLR4(
39+
${sql_grammar}
40+
-no-listener
41+
-package NALPAnsiAntlr4
42+
-lib .
43+
-o ${antlr_output}
44+
IN ${sql_grammar} ${antlr_templates}/Cpp/Cpp.stg ${antlr_templates}/Cpp/Files.stg
45+
OUT SQLv1Antlr4Parser.cpp SQLv1Antlr4Lexer.cpp SQLv1Antlr4Parser.h SQLv1Antlr4Lexer.h
46+
OUTPUT_INCLUDES
47+
${PROTOBUF_HEADER_PATH}/SQLv1Parser.pb.main.h
48+
${STG_INCLUDES}
49+
CWD ${antlr_output}
50+
)
51+
52+
END()

‎ydb/library/yql/parser/proto_ast/gen/v1_antlr4/epilogue.cmake

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
set(GRAMMAR_STRING_CORE_SINGLE "~(['\\]) | (BACKSLASH .)")
2+
set(GRAMMAR_STRING_CORE_DOUBLE "~([\"\\]) | (BACKSLASH .)")
3+
set(GRAMMAR_MULTILINE_COMMENT_CORE ".")
4+
configure_file(
5+
${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/SQLv1Antlr4.g.in
6+
${CMAKE_BINARY_DIR}/ydb/library/yql/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4.g
7+
)
8+
9+

‎ydb/library/yql/parser/proto_ast/gen/v1_antlr4/ya.make

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
LIBRARY()
2+
3+
PEERDIR (
4+
ydb/library/yql/parser/proto_ast/gen/v1_proto_split
5+
)
6+
7+
SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR})
8+
SET(antlr_templates ${antlr_output}/org/antlr/v4/tool/templates/codegen)
9+
SET(sql_grammar ${antlr_output}/SQLv1Antlr4.g)
10+
11+
SET(ANTLR_PACKAGE_NAME NSQLv1Generated)
12+
SET(PROTOBUF_HEADER_PATH ydb/library/yql/parser/proto_ast/gen/v1_proto_split)
13+
SET(PROTOBUF_SUFFIX_PATH .pb.main.h)
14+
15+
SET(LEXER_PARSER_NAMESPACE NALPDefaultAntlr4)
16+
17+
SET(GRAMMAR_STRING_CORE_SINGLE "\"~(['#BACKSLASH#]) | (BACKSLASH .)\"")
18+
SET(GRAMMAR_STRING_CORE_DOUBLE "\"~([#DOUBLE_QUOTE##BACKSLASH#]) | (BACKSLASH .)\"")
19+
SET(GRAMMAR_MULTILINE_COMMENT_CORE "\".\"")
20+
21+
CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg.in ${antlr_templates}/Cpp/Cpp.stg)
22+
CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg.in ${antlr_templates}/Cpp/Files.stg)
23+
24+
IF(EXPORT_CMAKE)
25+
MANUAL_GENERATION(${sql_grammar})
26+
ELSE()
27+
CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/sql/v1/SQLv1Antlr4.g.in ${sql_grammar})
28+
ENDIF()
29+
30+
NO_COMPILER_WARNINGS()
31+
32+
ADDINCL(
33+
GLOBAL contrib/libs/antlr4_cpp_runtime/src
34+
)
35+
36+
INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/ya.make.incl)
37+
38+
RUN_ANTLR4(
39+
${sql_grammar}
40+
-no-listener
41+
-package NALPDefaultAntlr4
42+
-lib .
43+
-o ${antlr_output}
44+
IN ${sql_grammar} ${antlr_templates}/Cpp/Cpp.stg ${antlr_templates}/Cpp/Files.stg
45+
OUT SQLv1Antlr4Parser.cpp SQLv1Antlr4Lexer.cpp SQLv1Antlr4Parser.h SQLv1Antlr4Lexer.h
46+
OUTPUT_INCLUDES
47+
${PROTOBUF_HEADER_PATH}/SQLv1Parser.pb.main.h
48+
${STG_INCLUDES}
49+
CWD ${antlr_output}
50+
)
51+
52+
END()

0 commit comments

Comments
 (0)