From a91886f2376dab9ead56a76f16f26417186c2a48 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 13 Sep 2022 18:43:28 +0800 Subject: [PATCH 1/7] Remove Eq requirement for TableMeta Signed-off-by: Xuanwo --- src/meta/app/src/schema/table.rs | 8 ++++---- src/meta/app/src/share/share.rs | 2 +- src/query/planners/src/plan_read_datasource.rs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/meta/app/src/schema/table.rs b/src/meta/app/src/schema/table.rs index d66f7a9388538..ab38de36d48fb 100644 --- a/src/meta/app/src/schema/table.rs +++ b/src/meta/app/src/schema/table.rs @@ -133,7 +133,7 @@ impl Display for TableIdListKey { } } -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Eq, PartialEq, Default)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Default)] pub struct TableInfo { pub ident: TableIdent, @@ -168,7 +168,7 @@ pub struct TableStatistics { /// The essential state that defines what a table is. /// /// It is what a meta store just needs to save. -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Eq, PartialEq)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] pub struct TableMeta { pub schema: Arc, pub engine: String, @@ -342,7 +342,7 @@ impl Display for TableIdList { } } -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] pub struct CreateTableReq { pub if_not_exists: bool, pub name_ident: TableNameIdent, @@ -497,7 +497,7 @@ pub struct UpsertTableOptionReq { pub options: HashMap>, } -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] pub struct UpdateTableMetaReq { pub table_id: u64, pub seq: MatchSeq, diff --git a/src/meta/app/src/share/share.rs b/src/meta/app/src/share/share.rs index 8c4cfdf6079ec..f275bf7fed7fc 100644 --- a/src/meta/app/src/share/share.rs +++ b/src/meta/app/src/share/share.rs @@ -161,7 +161,7 @@ impl Display for ShareGrantObjectName { } } -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] pub enum ShareGrantObjectSeqAndId { // db_meta_seq, db_id, DatabaseMeta Database(u64, u64, DatabaseMeta), diff --git a/src/query/planners/src/plan_read_datasource.rs b/src/query/planners/src/plan_read_datasource.rs index 643c95e6fe23c..784282e713a83 100644 --- a/src/query/planners/src/plan_read_datasource.rs +++ b/src/query/planners/src/plan_read_datasource.rs @@ -27,7 +27,7 @@ use crate::Projection; use crate::StageTableInfo; use crate::Statistics; -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] pub enum SourceInfo { // Normal table source, `fuse/system`. TableSource(TableInfo), From 808f5049eaadd41290b5991156ed1ab84e405f89 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 13 Sep 2022 20:13:15 +0800 Subject: [PATCH 2/7] Migrate expresion parser Signed-off-by: Xuanwo --- Cargo.lock | 1 + src/query/legacy-parser/Cargo.toml | 1 + src/query/legacy-parser/src/expression.rs | 84 +++++++++++++++++++ src/query/legacy-parser/src/lib.rs | 8 +- .../legacy-parser/src/parser/expr_parser.rs | 37 -------- .../src/parser/expression_parser.rs | 34 -------- src/query/legacy-parser/src/parser/mod.rs | 19 ----- src/query/service/src/sql/sql_parser.rs | 6 -- src/query/storages/fuse/src/fuse_table.rs | 4 +- .../clustering_informations/table_args.rs | 5 +- 10 files changed, 94 insertions(+), 105 deletions(-) create mode 100644 src/query/legacy-parser/src/expression.rs delete mode 100644 src/query/legacy-parser/src/parser/expr_parser.rs delete mode 100644 src/query/legacy-parser/src/parser/expression_parser.rs delete mode 100644 src/query/legacy-parser/src/parser/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 9adc5bf9d6508..f8768920ae25a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1391,6 +1391,7 @@ name = "common-legacy-parser" version = "0.1.0" dependencies = [ "async-trait", + "common-ast", "common-datavalues", "common-exception", "common-functions", diff --git a/src/query/legacy-parser/Cargo.toml b/src/query/legacy-parser/Cargo.toml index 0f7f29f7e8706..0191ed6dc8d6c 100644 --- a/src/query/legacy-parser/Cargo.toml +++ b/src/query/legacy-parser/Cargo.toml @@ -13,6 +13,7 @@ common-datavalues = { path = "../datavalues" } common-exception = { path = "../../common/exception" } common-functions = { path = "../functions" } common-planners = { path = "../planners" } +common-ast = { path = "../ast" } async-trait = "0.1.56" sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "7f246e3" } diff --git a/src/query/legacy-parser/src/expression.rs b/src/query/legacy-parser/src/expression.rs new file mode 100644 index 0000000000000..3f0b8800b0a8e --- /dev/null +++ b/src/query/legacy-parser/src/expression.rs @@ -0,0 +1,84 @@ +// Copyright 2022 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use common_ast::ast::Expr; +use common_ast::parser::parse_comma_separated_exprs; +use common_ast::parser::token::Token; +use common_ast::parser::token::Tokenizer; +use common_ast::Backtrace; +use common_ast::Dialect; +use common_exception::ErrorCode; +use common_exception::Result; +use common_planners::Expression; + +/// unchecked_expressions_analyze will analyzer given expr str into `Expression`. +/// +/// `unchecked` means there is no correction checks for expr: +/// +/// - We don't if the expr is **safe**. +/// - We don't if the expr is **existed**. +/// - We don't if the expr is **valid**. +/// +/// We just convert the given expr into Expression, and hoping everything works. +/// +/// This function should only be used by very limited use-cases like fuse engines +/// `cluster_keys` which already checked by our parsers. +/// +/// **FBI WARNING: DON'T USE THIS FUNCTION UNLESS YOU KNOW WHAT I MEAN** +/// +/// Any usage to this function should be reviewed carefully. +/// +/// # Future Plan +/// +/// This function should be removed once our new expression framework is ready. +pub fn unchecked_expressions_analyze(expr: &str) -> Result> { + let tokens: Vec = Tokenizer::new(expr).collect::>>()?; + let backtrace = Backtrace::new(); + let exprs = parse_comma_separated_exprs(&tokens, Dialect::MySQL, &backtrace)?; + + exprs + .into_iter() + .map(unchecked_expression_analyze) + .collect() +} + +fn unchecked_expression_analyze(expr: Expr) -> Result { + match expr { + Expr::ColumnRef { + database, + table, + column, + .. + } => { + if database.is_none() && table.is_none() { + Ok(Expression::Column(column.to_string())) + } else if database.is_none() { + Ok(Expression::QualifiedColumn(vec![ + table.unwrap().to_string(), + column.to_string(), + ])) + } else { + Ok(Expression::QualifiedColumn(vec![ + database.unwrap().to_string(), + table.unwrap().to_string(), + column.to_string(), + ])) + } + } + _ => Err(ErrorCode::LogicalError(format!( + "Logical error: can't analyze {:?}", + expr + ))), + } +} diff --git a/src/query/legacy-parser/src/lib.rs b/src/query/legacy-parser/src/lib.rs index cecc2f6d8bd2a..c317df794bd74 100644 --- a/src/query/legacy-parser/src/lib.rs +++ b/src/query/legacy-parser/src/lib.rs @@ -14,13 +14,13 @@ #![deny(unused_crate_dependencies)] +mod analyzer; pub use analyzer::analyzer_expr_sync; pub use analyzer::analyzer_value_expr; -mod analyzer; -mod parser; +mod expression; +/// Read the `FBI WARNING` docs before use this function. +pub use expression::unchecked_expressions_analyze; pub mod sql_common; mod sql_dialect; -pub use parser::ExprParser; -pub use parser::ExpressionParser; pub use sql_dialect::SQLDialect; diff --git a/src/query/legacy-parser/src/parser/expr_parser.rs b/src/query/legacy-parser/src/parser/expr_parser.rs deleted file mode 100644 index 1d4c88d201969..0000000000000 --- a/src/query/legacy-parser/src/parser/expr_parser.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2022 Datafuse Labs. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use sqlparser::ast::Expr; -use sqlparser::dialect::MySqlDialect; -use sqlparser::parser::Parser; -use sqlparser::parser::ParserError; -use sqlparser::tokenizer::Token; -use sqlparser::tokenizer::Tokenizer; - -pub struct ExprParser; - -impl ExprParser { - pub fn parse_exprs(expr: &str) -> Result, ParserError> { - let dialect = &MySqlDialect {}; - let mut tokenizer = Tokenizer::new(dialect, expr); - let (tokens, position_map) = tokenizer.tokenize()?; - let mut parser = Parser::new(tokens, position_map, dialect); - - parser.expect_token(&Token::LParen)?; - let exprs = parser.parse_comma_separated(Parser::parse_expr)?; - parser.expect_token(&Token::RParen)?; - - Ok(exprs) - } -} diff --git a/src/query/legacy-parser/src/parser/expression_parser.rs b/src/query/legacy-parser/src/parser/expression_parser.rs deleted file mode 100644 index 3666b23007544..0000000000000 --- a/src/query/legacy-parser/src/parser/expression_parser.rs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2022 Datafuse Labs. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use common_exception::Result; -use common_planners::Expression; - -use crate::analyzer_expr_sync::ExpressionSyncAnalyzer; -use crate::ExprParser; - -pub struct ExpressionParser; - -impl ExpressionParser { - pub fn parse_exprs(expr: &str) -> Result> { - let exprs = ExprParser::parse_exprs(expr)?; - let analyzer = ExpressionSyncAnalyzer::create(); - - let results = exprs - .iter() - .map(|expr| analyzer.analyze(expr)) - .collect::>>(); - results - } -} diff --git a/src/query/legacy-parser/src/parser/mod.rs b/src/query/legacy-parser/src/parser/mod.rs deleted file mode 100644 index 7374eb03f5bd1..0000000000000 --- a/src/query/legacy-parser/src/parser/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2022 Datafuse Labs. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod expr_parser; -mod expression_parser; - -pub use expr_parser::ExprParser; -pub use expression_parser::ExpressionParser; diff --git a/src/query/service/src/sql/sql_parser.rs b/src/query/service/src/sql/sql_parser.rs index 3c9c96fb211c3..562d358203790 100644 --- a/src/query/service/src/sql/sql_parser.rs +++ b/src/query/service/src/sql/sql_parser.rs @@ -18,9 +18,7 @@ use std::time::Instant; use common_exception::ErrorCode; -use common_legacy_parser::ExprParser; use metrics::histogram; -use sqlparser::ast::Expr; use sqlparser::dialect::keywords::Keyword; use sqlparser::dialect::Dialect; use sqlparser::dialect::GenericDialect; @@ -84,10 +82,6 @@ impl<'a> DfParser<'a> { } } - pub fn parse_exprs(expr: &str) -> Result, ParserError> { - ExprParser::parse_exprs(expr) - } - /// Parse a SQL statement and produce a set of statements pub fn parse_sql_with_dialect( sql: &'a str, diff --git a/src/query/storages/fuse/src/fuse_table.rs b/src/query/storages/fuse/src/fuse_table.rs index e0b696ede4b36..71d611ff45845 100644 --- a/src/query/storages/fuse/src/fuse_table.rs +++ b/src/query/storages/fuse/src/fuse_table.rs @@ -26,7 +26,7 @@ use common_fuse_meta::meta::ClusterKey; use common_fuse_meta::meta::Statistics as FuseStatistics; use common_fuse_meta::meta::TableSnapshot; use common_fuse_meta::meta::Versioned; -use common_legacy_parser::ExpressionParser; +use common_legacy_parser::unchecked_expressions_analyze; use common_meta_app::schema::TableInfo; use common_planners::DeletePlan; use common_planners::Expression; @@ -76,7 +76,7 @@ impl FuseTable { let cluster_key_meta = table_info.meta.cluster_key(); let mut cluster_keys = Vec::new(); if let Some((_, order)) = &cluster_key_meta { - cluster_keys = ExpressionParser::parse_exprs(order)?; + cluster_keys = unchecked_expressions_analyze(order)?; } Ok(Box::new(FuseTable { diff --git a/src/query/storages/fuse/src/table_functions/clustering_informations/table_args.rs b/src/query/storages/fuse/src/table_functions/clustering_informations/table_args.rs index d0d37708fb054..ae0bba77a7c2d 100644 --- a/src/query/storages/fuse/src/table_functions/clustering_informations/table_args.rs +++ b/src/query/storages/fuse/src/table_functions/clustering_informations/table_args.rs @@ -14,11 +14,10 @@ use common_exception::ErrorCode; use common_exception::Result; -use common_legacy_parser::ExpressionParser; +use common_legacy_parser::unchecked_expressions_analyze; use common_planners::validate_expression; use common_planners::Expression; -// use crate::sql::PlanParser; use crate::table_functions::string_value; use crate::table_functions::TableArgs; use crate::FuseTable; @@ -42,7 +41,7 @@ pub fn parse_func_table_args(table_args: &TableArgs) -> Result<(String, String)> pub fn get_cluster_keys(table: &FuseTable, definition: &str) -> Result> { let cluster_keys = if !definition.is_empty() { let schema = table.schema(); - let exprs = ExpressionParser::parse_exprs(definition)?; + let exprs = unchecked_expressions_analyze(definition)?; for expr in exprs.iter() { validate_expression(expr, &schema)?; } From 96431781fba8b25da26b2ddb383fe486e2458ee1 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 13 Sep 2022 20:13:29 +0800 Subject: [PATCH 3/7] Revert "Remove Eq requirement for TableMeta" This reverts commit a91886f2376dab9ead56a76f16f26417186c2a48. --- src/meta/app/src/schema/table.rs | 8 ++++---- src/meta/app/src/share/share.rs | 2 +- src/query/planners/src/plan_read_datasource.rs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/meta/app/src/schema/table.rs b/src/meta/app/src/schema/table.rs index ab38de36d48fb..d66f7a9388538 100644 --- a/src/meta/app/src/schema/table.rs +++ b/src/meta/app/src/schema/table.rs @@ -133,7 +133,7 @@ impl Display for TableIdListKey { } } -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Default)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Eq, PartialEq, Default)] pub struct TableInfo { pub ident: TableIdent, @@ -168,7 +168,7 @@ pub struct TableStatistics { /// The essential state that defines what a table is. /// /// It is what a meta store just needs to save. -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Eq, PartialEq)] pub struct TableMeta { pub schema: Arc, pub engine: String, @@ -342,7 +342,7 @@ impl Display for TableIdList { } } -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub struct CreateTableReq { pub if_not_exists: bool, pub name_ident: TableNameIdent, @@ -497,7 +497,7 @@ pub struct UpsertTableOptionReq { pub options: HashMap>, } -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub struct UpdateTableMetaReq { pub table_id: u64, pub seq: MatchSeq, diff --git a/src/meta/app/src/share/share.rs b/src/meta/app/src/share/share.rs index f275bf7fed7fc..8c4cfdf6079ec 100644 --- a/src/meta/app/src/share/share.rs +++ b/src/meta/app/src/share/share.rs @@ -161,7 +161,7 @@ impl Display for ShareGrantObjectName { } } -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub enum ShareGrantObjectSeqAndId { // db_meta_seq, db_id, DatabaseMeta Database(u64, u64, DatabaseMeta), diff --git a/src/query/planners/src/plan_read_datasource.rs b/src/query/planners/src/plan_read_datasource.rs index 784282e713a83..643c95e6fe23c 100644 --- a/src/query/planners/src/plan_read_datasource.rs +++ b/src/query/planners/src/plan_read_datasource.rs @@ -27,7 +27,7 @@ use crate::Projection; use crate::StageTableInfo; use crate::Statistics; -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub enum SourceInfo { // Normal table source, `fuse/system`. TableSource(TableInfo), From db869b8e990c7fe2d0cdbcc213926fa634d07b66 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 13 Sep 2022 20:18:47 +0800 Subject: [PATCH 4/7] Remove not used expr parser Signed-off-by: Xuanwo --- Cargo.lock | 1 - src/query/legacy-parser/Cargo.toml | 1 - .../src/analyzer/analyzer_expr_sync.rs | 308 ------------------ 3 files changed, 310 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f8768920ae25a..a4ec54a43a236 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1394,7 +1394,6 @@ dependencies = [ "common-ast", "common-datavalues", "common-exception", - "common-functions", "common-planners", "sqlparser", ] diff --git a/src/query/legacy-parser/Cargo.toml b/src/query/legacy-parser/Cargo.toml index 0191ed6dc8d6c..730d40fd3d08d 100644 --- a/src/query/legacy-parser/Cargo.toml +++ b/src/query/legacy-parser/Cargo.toml @@ -11,7 +11,6 @@ test = false [dependencies] common-datavalues = { path = "../datavalues" } common-exception = { path = "../../common/exception" } -common-functions = { path = "../functions" } common-planners = { path = "../planners" } common-ast = { path = "../ast" } diff --git a/src/query/legacy-parser/src/analyzer/analyzer_expr_sync.rs b/src/query/legacy-parser/src/analyzer/analyzer_expr_sync.rs index 808914be27f67..f555848885f14 100644 --- a/src/query/legacy-parser/src/analyzer/analyzer_expr_sync.rs +++ b/src/query/legacy-parser/src/analyzer/analyzer_expr_sync.rs @@ -14,11 +14,8 @@ use async_trait::async_trait; use common_datavalues::prelude::*; -use common_datavalues::type_coercion::merge_types; use common_exception::ErrorCode; use common_exception::Result; -use common_functions::aggregates::AggregateFunctionFactory; -use common_planners::Expression; use sqlparser::ast::BinaryOperator; use sqlparser::ast::DataType as AstDataType; use sqlparser::ast::DateTimeField; @@ -33,312 +30,7 @@ use sqlparser::ast::UnaryOperator; use sqlparser::ast::Value; use sqlparser::ast::WindowSpec; -use crate::analyzer_value_expr::ValueExprAnalyzer; use crate::sql_common::SQLCommon; -use crate::sql_dialect::SQLDialect; - -#[derive(Clone)] -pub struct ExpressionSyncAnalyzer {} - -impl ExpressionSyncAnalyzer { - pub fn create() -> ExpressionSyncAnalyzer { - ExpressionSyncAnalyzer {} - } - - pub fn analyze(&self, expr: &Expr) -> Result { - let mut stack = Vec::new(); - - // Build RPN for expr. Because async function unsupported recursion - for rpn_item in &ExprRPNBuilder::build(expr)? { - match rpn_item { - ExprRPNItem::Value(v) => Self::analyze_value(v, &mut stack, SQLDialect::MySQL)?, - ExprRPNItem::Identifier(v) => self.analyze_identifier(v, &mut stack)?, - ExprRPNItem::QualifiedIdentifier(v) => self.analyze_identifiers(v, &mut stack)?, - ExprRPNItem::Function(v) => self.analyze_function(v, &mut stack)?, - ExprRPNItem::Cast(v, pg_style) => self.analyze_cast(v, *pg_style, &mut stack)?, - ExprRPNItem::Between(negated) => self.analyze_between(*negated, &mut stack)?, - ExprRPNItem::InList(v) => self.analyze_inlist(v, &mut stack)?, - ExprRPNItem::MapAccess(v) => self.analyze_map_access(v, &mut stack)?, - ExprRPNItem::Array(v) => self.analyze_array(*v, &mut stack)?, - - _ => { - return Err(ErrorCode::LogicalError(format!( - "Logical error: can't analyze {:?} in sync mode, it's a bug", - expr - ))); - } - } - } - - match stack.len() { - 1 => Ok(stack.remove(0)), - _ => Err(ErrorCode::LogicalError( - "Logical error: this is expr rpn bug.", - )), - } - } - - pub fn analyze_function_arg(&self, arg_expr: &FunctionArgExpr) -> Result { - match arg_expr { - FunctionArgExpr::Expr(expr) => self.analyze(expr), - FunctionArgExpr::Wildcard => Ok(Expression::Wildcard), - FunctionArgExpr::QualifiedWildcard(_) => Err(ErrorCode::SyntaxException(std::format!( - "Unsupported arg statement: {}", - arg_expr - ))), - } - } - - fn analyze_value( - value: &Value, - args: &mut Vec, - typ: impl Into, - ) -> Result<()> { - args.push(ValueExprAnalyzer::analyze(value, typ)?); - Ok(()) - } - - fn analyze_inlist(&self, info: &InListInfo, args: &mut Vec) -> Result<()> { - let mut list = Vec::with_capacity(info.list_size); - for _ in 0..info.list_size { - match args.pop() { - None => { - return Err(ErrorCode::LogicalError("It's a bug.")); - } - Some(arg) => { - list.insert(0, arg); - } - } - } - - let expr = args - .pop() - .ok_or_else(|| ErrorCode::LogicalError("It's a bug."))?; - list.insert(0, expr); - - let op = if info.negated { - "NOT_IN".to_string() - } else { - "IN".to_string() - }; - - args.push(Expression::ScalarFunction { op, args: list }); - Ok(()) - } - - fn analyze_function(&self, info: &FunctionExprInfo, args: &mut Vec) -> Result<()> { - let mut arguments = Vec::with_capacity(info.args_count); - for _ in 0..info.args_count { - match args.pop() { - None => { - return Err(ErrorCode::LogicalError("It's a bug.")); - } - Some(arg) => { - arguments.insert(0, arg); - } - } - } - - args.push( - match AggregateFunctionFactory::instance().check(&info.name) { - true => { - return Err(ErrorCode::LogicalError( - "Unsupport aggregate function, it's a bug.", - )); - } - false => match info.kind { - OperatorKind::Unary => Self::unary_function(info, &arguments), - OperatorKind::Binary => Self::binary_function(info, &arguments), - OperatorKind::Other => Self::other_function(info, &arguments), - }, - }?, - ); - Ok(()) - } - - fn other_function(info: &FunctionExprInfo, args: &[Expression]) -> Result { - let op = info.name.clone(); - let arguments = args.to_owned(); - Ok(Expression::ScalarFunction { - op, - args: arguments, - }) - } - - fn unary_function(info: &FunctionExprInfo, args: &[Expression]) -> Result { - match args.is_empty() { - true => Err(ErrorCode::LogicalError("Unary operator must be one child.")), - false => Ok(Expression::UnaryExpression { - op: info.name.clone(), - expr: Box::new(args[0].to_owned()), - }), - } - } - - fn binary_function(info: &FunctionExprInfo, args: &[Expression]) -> Result { - let op = info.name.clone(); - match args.len() < 2 { - true => Err(ErrorCode::LogicalError( - "Binary operator must be two children.", - )), - false => Ok(Expression::BinaryExpression { - op, - left: Box::new(args[0].to_owned()), - right: Box::new(args[1].to_owned()), - }), - } - } - - fn analyze_identifier(&self, ident: &Ident, arguments: &mut Vec) -> Result<()> { - let column_name = ident.clone().value; - arguments.push(Expression::Column(column_name)); - Ok(()) - } - - fn analyze_identifiers(&self, idents: &[Ident], arguments: &mut Vec) -> Result<()> { - let mut names = Vec::with_capacity(idents.len()); - - for ident in idents { - names.push(ident.clone().value); - } - - arguments.push(Expression::QualifiedColumn(names)); - Ok(()) - } - - fn analyze_cast( - &self, - data_type: &DataTypeImpl, - pg_style: bool, - args: &mut Vec, - ) -> Result<()> { - match args.pop() { - None => Err(ErrorCode::LogicalError( - "Cast operator must be one children.", - )), - Some(inner_expr) => { - args.push(Expression::Cast { - expr: Box::new(inner_expr), - data_type: data_type.clone(), - pg_style, - }); - Ok(()) - } - } - } - - fn analyze_between(&self, negated: bool, args: &mut Vec) -> Result<()> { - if args.len() < 3 { - return Err(ErrorCode::SyntaxException( - "Between must be a ternary expression.", - )); - } - - let s_args = args.split_off(args.len() - 3); - let expression = s_args[0].clone(); - let low_expression = s_args[1].clone(); - let high_expression = s_args[2].clone(); - - match negated { - false => args.push( - expression - .gt_eq(low_expression) - .and(expression.lt_eq(high_expression)), - ), - true => args.push( - expression - .lt(low_expression) - .or(expression.gt(high_expression)), - ), - }; - - Ok(()) - } - - fn analyze_map_access(&self, keys: &[Value], args: &mut Vec) -> Result<()> { - match args.pop() { - None => Err(ErrorCode::LogicalError( - "MapAccess operator must be one children.", - )), - Some(inner_expr) => { - let path_name: String = keys - .iter() - .enumerate() - .map(|(i, k)| match k { - k @ Value::Number(_, _) => format!("[{}]", k), - Value::SingleQuotedString(s) => format!("[\"{}\"]", s), - Value::ColonString(s) => { - if i == 0 { - s.to_string() - } else { - format!(":{}", s) - } - } - Value::PeriodString(s) => format!(".{}", s), - _ => format!("[{}]", k), - }) - .collect(); - - let name = match keys[0] { - Value::ColonString(_) => format!("{}:{}", inner_expr.column_name(), path_name), - _ => format!("{}{}", inner_expr.column_name(), path_name), - }; - let path = - Expression::create_literal(DataValue::String(path_name.as_bytes().to_vec())); - let arguments = vec![inner_expr, path]; - - args.push(Expression::MapAccess { - name, - args: arguments, - }); - Ok(()) - } - } - } - - fn analyze_array(&self, nums: usize, args: &mut Vec) -> Result<()> { - let mut values = Vec::with_capacity(nums); - let mut types = Vec::with_capacity(nums); - for _ in 0..nums { - match args.pop() { - None => { - break; - } - Some(inner_expr) => { - if let Expression::Literal { - value, data_type, .. - } = inner_expr - { - values.push(value); - types.push(data_type); - } - } - }; - } - if values.len() != nums { - return Err(ErrorCode::LogicalError(format!( - "Array must have {} children.", - nums - ))); - } - let inner_type = if types.is_empty() { - NullType::new_impl() - } else { - types - .iter() - .fold(Ok(types[0].clone()), |acc, v| merge_types(&acc?, v)) - .map_err(|e| ErrorCode::LogicalError(e.message()))? - }; - values.reverse(); - - let array_value = Expression::create_literal_with_type( - DataValue::Array(values), - ArrayType::new_impl(inner_type), - ); - args.push(array_value); - Ok(()) - } -} pub enum OperatorKind { Unary, From 454f1c8b7c606c6aa182e8f8175a1105bf59e10a Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 13 Sep 2022 20:24:42 +0800 Subject: [PATCH 5/7] Format deps Signed-off-by: Xuanwo --- src/query/legacy-parser/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/legacy-parser/Cargo.toml b/src/query/legacy-parser/Cargo.toml index 730d40fd3d08d..d9a7014da0712 100644 --- a/src/query/legacy-parser/Cargo.toml +++ b/src/query/legacy-parser/Cargo.toml @@ -9,10 +9,10 @@ doctest = false test = false [dependencies] +common-ast = { path = "../ast" } common-datavalues = { path = "../datavalues" } common-exception = { path = "../../common/exception" } common-planners = { path = "../planners" } -common-ast = { path = "../ast" } async-trait = "0.1.56" sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "7f246e3" } From 8951fe7f65585b474c738c97d112ea4c5976a3d1 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 13 Sep 2022 20:46:43 +0800 Subject: [PATCH 6/7] Fix tests Signed-off-by: Xuanwo --- src/query/legacy-parser/src/expression.rs | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/query/legacy-parser/src/expression.rs b/src/query/legacy-parser/src/expression.rs index 3f0b8800b0a8e..04951419bbb3c 100644 --- a/src/query/legacy-parser/src/expression.rs +++ b/src/query/legacy-parser/src/expression.rs @@ -13,11 +13,13 @@ // limitations under the License. use common_ast::ast::Expr; +use common_ast::ast::Literal; use common_ast::parser::parse_comma_separated_exprs; use common_ast::parser::token::Token; use common_ast::parser::token::Tokenizer; use common_ast::Backtrace; use common_ast::Dialect; +use common_datavalues::DataValue; use common_exception::ErrorCode; use common_exception::Result; use common_planners::Expression; @@ -76,6 +78,33 @@ fn unchecked_expression_analyze(expr: Expr) -> Result { ])) } } + Expr::BinaryOp { + op, left, right, .. + } => Ok(Expression::BinaryExpression { + left: Box::new(unchecked_expression_analyze(*left)?), + op: op.to_string(), + right: Box::new(unchecked_expression_analyze(*right)?), + }), + Expr::Literal { lit, .. } => { + let value = match lit { + Literal::Integer(uint) => DataValue::UInt64(uint), + Literal::Float(float) => DataValue::Float64(float), + Literal::String(string) => DataValue::String(string.as_bytes().to_vec()), + Literal::Boolean(boolean) => DataValue::Boolean(boolean), + Literal::Null => DataValue::Null, + _ => Err(ErrorCode::SemanticError(format!( + "Unsupported literal value: {lit}" + )))?, + }; + + let data_type = value.data_type(); + + Ok(Expression::Literal { + value, + column_name: None, + data_type, + }) + } _ => Err(ErrorCode::LogicalError(format!( "Logical error: can't analyze {:?}", expr From 52ee25b3ecae754e33322cc05064b212450ccd26 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 13 Sep 2022 21:03:05 +0800 Subject: [PATCH 7/7] Fix test Signed-off-by: Xuanwo --- src/query/legacy-parser/src/expression.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/query/legacy-parser/src/expression.rs b/src/query/legacy-parser/src/expression.rs index 04951419bbb3c..b02c69bc1c491 100644 --- a/src/query/legacy-parser/src/expression.rs +++ b/src/query/legacy-parser/src/expression.rs @@ -105,6 +105,16 @@ fn unchecked_expression_analyze(expr: Expr) -> Result { data_type, }) } + Expr::Tuple { exprs, .. } => { + let mut args = Vec::with_capacity(exprs.len()); + for expr in exprs { + args.push(unchecked_expression_analyze(expr)?); + } + Ok(Expression::ScalarFunction { + op: "tuple".to_string(), + args, + }) + } _ => Err(ErrorCode::LogicalError(format!( "Logical error: can't analyze {:?}", expr