Introduced lexer & parser interfaces

commit_hash:fee365c90a176dd33a967cee20994b21d530080c

Author: vitstn

yql/essentials/parser/lexer_common/lexer.cpp

@@ -0,0 +1,49 @@
+#include "lexer.h"
+#include <yql/essentials/public/issue/yql_issue.h>
+
+#include <util/string/builder.h>
+
+namespace NSQLTranslation {
+
+namespace {
+
+class TDummyLexer : public ILexer {
+public:
+    TDummyLexer(const TString& name)
+        : Name_(name)
+    {}
+
+    bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) final {
+        Y_UNUSED(query);
+        Y_UNUSED(queryName);
+        Y_UNUSED(onNextToken);
+        Y_UNUSED(maxErrors);
+        issues.AddIssue(NYql::TIssue({}, TStringBuilder() << "Lexer " << Name_ << " is not supported"));
+        return false;
+    }
+
+private:
+    const TString Name_;
+};
+
+class TDummyFactory: public ILexerFactory {
+public:
+    TDummyFactory(const TString& name)
+        : Name_(name)
+    {}
+
+    ILexer::TPtr MakeLexer() const final {
+        return MakeHolder<TDummyLexer>(Name_);
+    }
+
+private:
+    const TString Name_;
+};
+
+}
+
+TLexerFactoryPtr MakeDummyLexerFactory(const TString& name) {
+    return MakeIntrusive<TDummyFactory>(name);
+}
+
+}

yql/essentials/parser/lexer_common/lexer.h

@@ -39,5 +39,16 @@ using TParsedTokenList = TVector<TParsedToken>;
 IOutputStream& OutputTokens(IOutputStream& out, TParsedTokenList::const_iterator begin, TParsedTokenList::const_iterator end);
 bool Tokenize(ILexer& lexer, const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors);
 
+class ILexerFactory : public TThrRefBase {
+public:
+    virtual ~ILexerFactory() = default;
+
+    virtual ILexer::TPtr MakeLexer() const = 0;
+};
+
+using TLexerFactoryPtr = TIntrusivePtr<ILexerFactory>;
+
+TLexerFactoryPtr MakeDummyLexerFactory(const TString& name);
+
 }
 

yql/essentials/parser/lexer_common/ya.make

@@ -7,6 +7,7 @@ PEERDIR(
 SRCS(
     tokens.cpp
     lexer.h
+    lexer.cpp
     hints.cpp
     hints.h
     parse_hints_impl.cpp

yql/essentials/parser/proto_ast/common.h

@@ -65,5 +65,24 @@ namespace NProtoAST {
         IOutputStream& Err;
         TString Name;
     };
-
 } // namespace NProtoAST
+
+namespace NSQLTranslation {
+    class IParser {
+    public:
+        virtual ~IParser() = default;
+
+        virtual google::protobuf::Message* Parse(
+            const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err,
+            google::protobuf::Arena* arena) = 0;
+    };
+
+    class IParserFactory : public TThrRefBase {
+    public:
+        virtual ~IParserFactory() = default;
+
+        virtual std::unique_ptr<IParser> MakeParser() const = 0;
+    };
+
+    using TParserFactoryPtr = TIntrusivePtr<IParserFactory>;
+} // namespace NSQLTranslation

yql/essentials/sql/v1/lexer/antlr3/lexer.cpp

@@ -0,0 +1,40 @@
+#include "lexer.h"
+#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h>
+#include <yql/essentials/public/issue/yql_issue.h>
+#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h>
+#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h>
+
+namespace NALPDefault {
+extern ANTLR_UINT8 *SQLv1ParserTokenNames[];
+}
+
+namespace NSQLTranslationV1 {
+
+namespace {
+
+class TLexer: public NSQLTranslation::ILexer {
+public:
+    bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) final {
+        NYql::TIssues newIssues;
+        NSQLTranslation::TErrorCollectorOverIssues collector(newIssues, maxErrors, queryName);
+        NProtoAST::TLexerTokensCollector3<NALPDefault::SQLv1Lexer> tokensCollector(query, (const char**)NALPDefault::SQLv1ParserTokenNames, queryName);
+        tokensCollector.CollectTokens(collector, onNextToken);
+        issues.AddIssues(newIssues);
+        return !AnyOf(newIssues.begin(), newIssues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; });
+    }
+};
+
+class TFactory: public NSQLTranslation::ILexerFactory {
+public:
+    THolder<NSQLTranslation::ILexer> MakeLexer() const final {
+        return MakeHolder<TLexer>();
+    }
+};
+
+}
+
+NSQLTranslation::TLexerFactoryPtr MakeAntlr3LexerFactory() {
+    return MakeIntrusive<TFactory>();
+}
+
+}

yql/essentials/sql/v1/lexer/antlr3/lexer.h

@@ -0,0 +1,8 @@
+#pragma once
+#include <yql/essentials/parser/lexer_common/lexer.h>
+
+namespace NSQLTranslationV1 {
+
+NSQLTranslation::TLexerFactoryPtr MakeAntlr3LexerFactory();
+
+}

yql/essentials/sql/v1/lexer/antlr3/ya.make

@@ -0,0 +1,14 @@
+LIBRARY()
+
+PEERDIR(
+    yql/essentials/parser/lexer_common
+    yql/essentials/public/issue
+    yql/essentials/parser/proto_ast/collect_issues
+    yql/essentials/parser/proto_ast/gen/v1
+)
+
+SRCS(
+    lexer.cpp
+)
+
+END()

yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.cpp

@@ -0,0 +1,40 @@
+#include "lexer.h"
+#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Lexer.h>
+#include <yql/essentials/public/issue/yql_issue.h>
+#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h>
+#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h>
+
+namespace NALPAnsi {
+extern ANTLR_UINT8 *SQLv1ParserTokenNames[];
+}
+
+namespace NSQLTranslationV1 {
+
+namespace {
+
+class TLexer: public NSQLTranslation::ILexer {
+public:
+    bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) final {
+        NYql::TIssues newIssues;
+        NSQLTranslation::TErrorCollectorOverIssues collector(newIssues, maxErrors, queryName);
+        NProtoAST::TLexerTokensCollector3<NALPAnsi::SQLv1Lexer> tokensCollector(query, (const char**)NALPAnsi::SQLv1ParserTokenNames, queryName);
+        tokensCollector.CollectTokens(collector, onNextToken);
+        issues.AddIssues(newIssues);
+        return !AnyOf(newIssues.begin(), newIssues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; });
+    }
+};
+
+class TFactory: public NSQLTranslation::ILexerFactory {
+public:
+    THolder<NSQLTranslation::ILexer> MakeLexer() const final {
+        return MakeHolder<TLexer>();
+    }
+};
+
+}
+
+NSQLTranslation::TLexerFactoryPtr MakeAntlr3AnsiLexerFactory() {
+    return MakeIntrusive<TFactory>();
+}
+
+}

yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h

@@ -0,0 +1,8 @@
+#pragma once
+#include <yql/essentials/parser/lexer_common/lexer.h>
+
+namespace NSQLTranslationV1 {
+
+NSQLTranslation::TLexerFactoryPtr MakeAntlr3AnsiLexerFactory();
+
+}

yql/essentials/sql/v1/lexer/antlr3_ansi/ya.make

@@ -0,0 +1,14 @@
+LIBRARY()
+
+PEERDIR(
+    yql/essentials/parser/lexer_common
+    yql/essentials/public/issue
+    yql/essentials/parser/proto_ast/collect_issues
+    yql/essentials/parser/proto_ast/gen/v1_ansi
+)
+
+SRCS(
+    lexer.cpp
+)
+
+END()