From e8804530865ad9cc3f9666cf1463f7a8d0b7bd5a Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 19 Jun 2025 14:54:40 +0800 Subject: [PATCH 01/37] add MaterializedCTE plan --- .../builders/builder_materialized_cte.rs | 26 ++++++++++++++ .../service/src/pipelines/builders/mod.rs | 1 + .../service/src/pipelines/pipeline_builder.rs | 2 ++ .../transform_recursive_cte_source.rs | 4 +++ src/query/sql/src/executor/format.rs | 10 ++++++ src/query/sql/src/executor/physical_plan.rs | 21 ++++++++++- .../sql/src/executor/physical_plan_visitor.rs | 17 +++++++++ .../sql/src/executor/physical_plans/mod.rs | 2 ++ .../physical_materialized_cte.rs | 36 +++++++++++++++++++ 9 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 src/query/service/src/pipelines/builders/builder_materialized_cte.rs create mode 100644 src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs new file mode 100644 index 0000000000000..321eccd3e0ccd --- /dev/null +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -0,0 +1,26 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_exception::Result; +use databend_common_sql::executor::physical_plans::MaterializedCTE; + +use crate::pipelines::PipelineBuilder; + +impl PipelineBuilder { + pub(crate) fn build_materialized_cte(&mut self, cte: &MaterializedCTE) -> Result<()> { + self.build_pipeline(&cte.left)?; + self.build_pipeline(&cte.right)?; + Ok(()) + } +} \ No newline at end of file diff --git a/src/query/service/src/pipelines/builders/mod.rs b/src/query/service/src/pipelines/builders/mod.rs index 552580bc7c467..052416707a126 100644 --- a/src/query/service/src/pipelines/builders/mod.rs +++ b/src/query/service/src/pipelines/builders/mod.rs @@ -49,6 +49,7 @@ mod builder_union_all; mod builder_window; mod merge_into_join_optimizations; mod transform_builder; +mod builder_materialized_cte; pub use builder_replace_into::RawValueSource; pub use builder_replace_into::ValueSource; diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index dd9f9d3af7a17..c7f9d773d5e55 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -292,6 +292,8 @@ impl PipelineBuilder { PhysicalPlan::Exchange(_) => Err(ErrorCode::Internal( "Invalid physical plan with PhysicalPlan::Exchange", )), + + PhysicalPlan::MaterializedCTE(cte) => self.build_materialized_cte(cte), }?; self.is_exchange_stack.pop(); diff --git a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs index d8596ddfc9e5f..14c353fd8c331 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs @@ -320,6 +320,10 @@ async fn create_memory_table_for_cte_scan( PhysicalPlan::AsyncFunction(plan) => { create_memory_table_for_cte_scan(ctx, plan.input.as_ref()).await?; } + PhysicalPlan::MaterializedCTE(plan) => { + create_memory_table_for_cte_scan(ctx, plan.left.as_ref()).await?; + create_memory_table_for_cte_scan(ctx, plan.right.as_ref()).await?; + } PhysicalPlan::TableScan(_) | PhysicalPlan::ConstantTableScan(_) | PhysicalPlan::ExpressionScan(_) diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index 1e3f8879339f3..95eaf484d59b7 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -527,6 +527,16 @@ fn to_format_tree( PhysicalPlan::BroadcastSink(_plan) => { Ok(FormatTreeNode::new("RuntimeFilterSink".to_string())) } + PhysicalPlan::MaterializedCTE(plan) => { + let mut children = Vec::new(); + append_profile_info(&mut children, profs, plan.plan_id); + children.push(to_format_tree(&plan.left, metadata, profs, context)?); + children.push(to_format_tree(&plan.right, metadata, profs, context)?); + Ok(FormatTreeNode::with_children( + "MaterializedCTE".to_string(), + children, + )) + } } } diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 33f29fabf5cd1..0c1c7f1c60d2a 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -71,6 +71,7 @@ use crate::executor::physical_plans::ReplaceAsyncSourcer; use crate::executor::physical_plans::ReplaceDeduplicate; use crate::executor::physical_plans::ReplaceInto; use crate::executor::physical_plans::RowFetch; +use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Shuffle; use crate::executor::physical_plans::Sort; use crate::executor::physical_plans::TableScan; @@ -161,6 +162,8 @@ pub enum PhysicalPlan { // broadcast BroadcastSource(BroadcastSource), BroadcastSink(BroadcastSink), + + MaterializedCTE(Box), } impl PhysicalPlan { @@ -422,6 +425,12 @@ impl PhysicalPlan { *next_id += 1; plan.input.adjust_plan_id(next_id); } + PhysicalPlan::MaterializedCTE(plan) => { + plan.plan_id = *next_id; + *next_id += 1; + plan.left.adjust_plan_id(next_id); + plan.right.adjust_plan_id(next_id); + } } } @@ -480,6 +489,7 @@ impl PhysicalPlan { PhysicalPlan::RecursiveCteScan(v) => v.plan_id, PhysicalPlan::BroadcastSource(v) => v.plan_id, PhysicalPlan::BroadcastSink(v) => v.plan_id, + PhysicalPlan::MaterializedCTE(v) => v.plan_id, } } @@ -537,6 +547,7 @@ impl PhysicalPlan { PhysicalPlan::ChunkAppendData(_) => todo!(), PhysicalPlan::ChunkMerge(_) => todo!(), PhysicalPlan::ChunkCommitInsert(_) => todo!(), + PhysicalPlan::MaterializedCTE(plan) => plan.output_schema(), } } @@ -600,6 +611,7 @@ impl PhysicalPlan { PhysicalPlan::ChunkCommitInsert(_) => "Commit".to_string(), PhysicalPlan::BroadcastSource(_) => "RuntimeFilterSource".to_string(), PhysicalPlan::BroadcastSink(_) => "RuntimeFilterSink".to_string(), + PhysicalPlan::MaterializedCTE(_) => "MaterializedCTE".to_string(), } } @@ -674,6 +686,9 @@ impl PhysicalPlan { CopyIntoTableSource::Query(v) => Box::new(std::iter::once(v.as_ref())), CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v.as_ref())), }, + PhysicalPlan::MaterializedCTE(plan) => Box::new( + std::iter::once(plan.left.as_ref()).chain(std::iter::once(plan.right.as_ref())), + ), } } @@ -747,6 +762,9 @@ impl PhysicalPlan { CopyIntoTableSource::Query(v) => Box::new(std::iter::once(v.as_mut())), CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v.as_mut())), }, + PhysicalPlan::MaterializedCTE(plan) => Box::new( + std::iter::once(plan.left.as_mut()).chain(std::iter::once(plan.right.as_mut())), + ), PhysicalPlan::BroadcastSink(plan) => Box::new(std::iter::once(plan.input.as_mut())), } } @@ -805,7 +823,8 @@ impl PhysicalPlan { | PhysicalPlan::ChunkMerge(_) | PhysicalPlan::ChunkCommitInsert(_) | PhysicalPlan::BroadcastSource(_) - | PhysicalPlan::BroadcastSink(_) => None, + | PhysicalPlan::BroadcastSink(_) + | PhysicalPlan::MaterializedCTE(_) => None, } } diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index c26807609bf81..926ffdb938827 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -61,6 +61,7 @@ use crate::executor::physical_plans::ReplaceAsyncSourcer; use crate::executor::physical_plans::ReplaceDeduplicate; use crate::executor::physical_plans::ReplaceInto; use crate::executor::physical_plans::RowFetch; +use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Shuffle; use crate::executor::physical_plans::Sort; use crate::executor::physical_plans::TableScan; @@ -124,6 +125,7 @@ pub trait PhysicalPlanReplacer { PhysicalPlan::ChunkCommitInsert(plan) => self.replace_chunk_commit_insert(plan), PhysicalPlan::BroadcastSource(plan) => self.replace_runtime_filter_source(plan), PhysicalPlan::BroadcastSink(plan) => self.replace_runtime_filter_sink(plan), + PhysicalPlan::MaterializedCTE(plan) => self.replace_materialized_cte(plan), } } @@ -642,6 +644,17 @@ pub trait PhysicalPlanReplacer { }, ))) } + + fn replace_materialized_cte(&mut self, plan: &MaterializedCTE) -> Result { + let left = self.replace(&plan.left)?; + let right = self.replace(&plan.right)?; + Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { + plan_id: plan.plan_id, + left: Box::new(left), + right: Box::new(right), + stat_info: plan.stat_info.clone(), + }))) + } } impl PhysicalPlan { @@ -794,6 +807,10 @@ impl PhysicalPlan { PhysicalPlan::BroadcastSink(plan) => { Self::traverse(&plan.input, pre_visit, visit, post_visit); } + PhysicalPlan::MaterializedCTE(plan) => { + Self::traverse(&plan.left, pre_visit, visit, post_visit); + Self::traverse(&plan.right, pre_visit, visit, post_visit); + } } post_visit(plan); } diff --git a/src/query/sql/src/executor/physical_plans/mod.rs b/src/query/sql/src/executor/physical_plans/mod.rs index 2e04928a2a55c..9f9ee83dce48c 100644 --- a/src/query/sql/src/executor/physical_plans/mod.rs +++ b/src/query/sql/src/executor/physical_plans/mod.rs @@ -53,6 +53,7 @@ mod physical_replace_async_source; mod physical_replace_deduplicate; mod physical_replace_into; mod physical_row_fetch; +mod physical_materialized_cte; mod physical_sort; mod physical_table_scan; mod physical_udf; @@ -106,6 +107,7 @@ pub use physical_replace_async_source::ReplaceAsyncSourcer; pub use physical_replace_deduplicate::*; pub use physical_replace_into::ReplaceInto; pub use physical_row_fetch::RowFetch; +pub use physical_materialized_cte::MaterializedCTE; pub use physical_sort::Sort; pub use physical_table_scan::TableScan; pub use physical_udf::Udf; diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs new file mode 100644 index 0000000000000..901d1eed99764 --- /dev/null +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -0,0 +1,36 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_exception::Result; +use databend_common_expression::DataSchemaRef; + +use crate::executor::explain::PlanStatsInfo; +use crate::executor::PhysicalPlan; + +/// This is a binary operator that executes its children in order (left to right), and returns the results of the right child +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct MaterializedCTE { + // A unique id of operator in a `PhysicalPlan` tree, only used for display. + pub plan_id: u32, + // Only used for explain + pub stat_info: Option, + pub left: Box, + pub right: Box, +} + +impl MaterializedCTE { + pub fn output_schema(&self) -> Result { + self.right.output_schema() + } +} From 466e1635c240527cda5b78fd9aea14a0604e45f5 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 19 Jun 2025 21:09:00 +0800 Subject: [PATCH 02/37] build pipeline --- .../builders/builder_materialized_cte.rs | 29 +++++++- .../service/src/pipelines/builders/mod.rs | 2 +- .../service/src/pipelines/pipeline_builder.rs | 4 ++ .../processors/transforms/materialized_cte.rs | 69 +++++++++++++++++++ .../pipelines/processors/transforms/mod.rs | 3 + src/query/sql/src/executor/physical_plan.rs | 2 +- .../sql/src/executor/physical_plan_visitor.rs | 3 +- .../sql/src/executor/physical_plans/mod.rs | 4 +- .../physical_materialized_cte.rs | 1 + 9 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 src/query/service/src/pipelines/processors/transforms/materialized_cte.rs diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 321eccd3e0ccd..385a07ccc8e44 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -15,12 +15,35 @@ use databend_common_exception::Result; use databend_common_sql::executor::physical_plans::MaterializedCTE; +use crate::pipelines::processors::transforms::MaterializedCteSink; use crate::pipelines::PipelineBuilder; - +use crate::sessions::QueryContext; impl PipelineBuilder { pub(crate) fn build_materialized_cte(&mut self, cte: &MaterializedCTE) -> Result<()> { - self.build_pipeline(&cte.left)?; + // init builder for cte pipeline + let sub_context = QueryContext::create_from(self.ctx.as_ref()); + let sub_builder = PipelineBuilder::create( + self.func_ctx.clone(), + self.settings.clone(), + sub_context, + self.main_pipeline.get_scopes(), + ); + + // build cte pipeline + let mut build_res = sub_builder.finalize(&cte.left)?; + build_res.main_pipeline.try_resize(1)?; + let (tx, rx) = tokio::sync::watch::channel(Default::default()); + self.cte_receivers.insert(cte.cte_name.clone(), rx); + build_res + .main_pipeline + .add_sink(|input| MaterializedCteSink::create(input, tx.clone()))?; + + // add cte pipeline to pipelines + self.pipelines.push(build_res.main_pipeline); + self.pipelines.extend(build_res.sources_pipelines); + + // build main pipeline self.build_pipeline(&cte.right)?; Ok(()) } -} \ No newline at end of file +} diff --git a/src/query/service/src/pipelines/builders/mod.rs b/src/query/service/src/pipelines/builders/mod.rs index 052416707a126..a229fa18d008d 100644 --- a/src/query/service/src/pipelines/builders/mod.rs +++ b/src/query/service/src/pipelines/builders/mod.rs @@ -30,6 +30,7 @@ mod builder_hilbert_partition; mod builder_insert_multi_table; mod builder_join; mod builder_limit; +mod builder_materialized_cte; mod builder_mutation; mod builder_mutation_manipulate; mod builder_mutation_organize; @@ -49,7 +50,6 @@ mod builder_union_all; mod builder_window; mod merge_into_join_optimizations; mod transform_builder; -mod builder_materialized_cte; pub use builder_replace_into::RawValueSource; pub use builder_replace_into::ValueSource; diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index c7f9d773d5e55..c44fefd55cfbb 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -27,9 +27,11 @@ use databend_common_pipeline_core::ExecutionInfo; use databend_common_pipeline_core::Pipeline; use databend_common_settings::Settings; use databend_common_sql::executor::PhysicalPlan; +use tokio::sync::watch::Receiver; use super::PipelineBuilderData; use crate::interpreters::CreateTableInterpreter; +use crate::pipelines::processors::transforms::MaterializedCteData; use crate::pipelines::processors::HashJoinBuildState; use crate::pipelines::processors::HashJoinState; use crate::pipelines::PipelineBuildResult; @@ -57,6 +59,7 @@ pub struct PipelineBuilder { pub(crate) is_exchange_stack: Vec, pub contain_sink_processor: bool, + pub cte_receivers: HashMap>>, } impl PipelineBuilder { @@ -79,6 +82,7 @@ impl PipelineBuilder { r_cte_scan_interpreters: vec![], contain_sink_processor: false, is_exchange_stack: vec![], + cte_receivers: HashMap::new(), } } diff --git a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs new file mode 100644 index 0000000000000..7d0d8e28c1018 --- /dev/null +++ b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs @@ -0,0 +1,69 @@ +use std::sync::Arc; + +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_expression::DataBlock; +use databend_common_pipeline_core::processors::InputPort; +use databend_common_pipeline_core::processors::ProcessorPtr; +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use databend_common_pipeline_sinks::Sink; +use databend_common_pipeline_sinks::Sinker; +use tokio::sync::watch::Sender; + +pub struct MaterializedCteSink { + sender: Sender>, + blocks: Vec, +} + +#[derive(Default)] +pub struct MaterializedCteData { + blocks: Vec, +} + +impl MaterializedCteData { + pub fn new(blocks: Vec) -> Self { + Self { blocks } + } +} + +impl MaterializedCteSink { + pub fn create( + input: Arc, + sender: Sender>, + ) -> Result { + Ok(ProcessorPtr::create(Sinker::create(input, Self { + blocks: vec![], + sender, + }))) + } +} + +impl Sink for MaterializedCteSink { + const NAME: &'static str = "MaterializedCteSink"; + + fn consume(&mut self, data_block: DataBlock) -> Result<()> { + self.blocks.push(data_block); + Ok(()) + } + + fn on_finish(&mut self) -> Result<()> { + self.sender + .send(Arc::new(MaterializedCteData::new(self.blocks.clone()))) + .map_err(|_| { + ErrorCode::Internal("Failed to send blocks to materialized cte consumer") + })?; + Ok(()) + } +} diff --git a/src/query/service/src/pipelines/processors/transforms/mod.rs b/src/query/service/src/pipelines/processors/transforms/mod.rs index 80966daa5fa8d..25f8dea06f2c9 100644 --- a/src/query/service/src/pipelines/processors/transforms/mod.rs +++ b/src/query/service/src/pipelines/processors/transforms/mod.rs @@ -16,6 +16,7 @@ pub mod aggregator; #[allow(dead_code)] mod broadcast; mod hash_join; +mod materialized_cte; pub(crate) mod range_join; mod runtime_pool; mod transform_add_computed_columns; @@ -46,6 +47,8 @@ mod window; pub use broadcast::BroadcastSinkProcessor; pub use broadcast::BroadcastSourceProcessor; pub use hash_join::*; +pub use materialized_cte::MaterializedCteData; +pub use materialized_cte::MaterializedCteSink; pub use transform_add_computed_columns::TransformAddComputedColumns; pub use transform_add_const_columns::TransformAddConstColumns; pub use transform_add_internal_columns::TransformAddInternalColumns; diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 0c1c7f1c60d2a..2bbf03bb7ef70 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -62,6 +62,7 @@ use crate::executor::physical_plans::ExpressionScan; use crate::executor::physical_plans::Filter; use crate::executor::physical_plans::HashJoin; use crate::executor::physical_plans::Limit; +use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Mutation; use crate::executor::physical_plans::ProjectSet; use crate::executor::physical_plans::RangeJoin; @@ -71,7 +72,6 @@ use crate::executor::physical_plans::ReplaceAsyncSourcer; use crate::executor::physical_plans::ReplaceDeduplicate; use crate::executor::physical_plans::ReplaceInto; use crate::executor::physical_plans::RowFetch; -use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Shuffle; use crate::executor::physical_plans::Sort; use crate::executor::physical_plans::TableScan; diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index 926ffdb938827..3973665cc64b1 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -52,6 +52,7 @@ use crate::executor::physical_plans::ExchangeSource; use crate::executor::physical_plans::Filter; use crate::executor::physical_plans::HashJoin; use crate::executor::physical_plans::Limit; +use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Mutation; use crate::executor::physical_plans::MutationSource; use crate::executor::physical_plans::ProjectSet; @@ -61,7 +62,6 @@ use crate::executor::physical_plans::ReplaceAsyncSourcer; use crate::executor::physical_plans::ReplaceDeduplicate; use crate::executor::physical_plans::ReplaceInto; use crate::executor::physical_plans::RowFetch; -use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Shuffle; use crate::executor::physical_plans::Sort; use crate::executor::physical_plans::TableScan; @@ -653,6 +653,7 @@ pub trait PhysicalPlanReplacer { left: Box::new(left), right: Box::new(right), stat_info: plan.stat_info.clone(), + cte_name: plan.cte_name.clone(), }))) } } diff --git a/src/query/sql/src/executor/physical_plans/mod.rs b/src/query/sql/src/executor/physical_plans/mod.rs index 9f9ee83dce48c..6275ef8b46771 100644 --- a/src/query/sql/src/executor/physical_plans/mod.rs +++ b/src/query/sql/src/executor/physical_plans/mod.rs @@ -38,6 +38,7 @@ mod physical_hash_join; mod physical_join; mod physical_join_filter; mod physical_limit; +mod physical_materialized_cte; mod physical_multi_table_insert; mod physical_mutation; mod physical_mutation_into_organize; @@ -53,7 +54,6 @@ mod physical_replace_async_source; mod physical_replace_deduplicate; mod physical_replace_into; mod physical_row_fetch; -mod physical_materialized_cte; mod physical_sort; mod physical_table_scan; mod physical_udf; @@ -91,6 +91,7 @@ pub use physical_join_filter::JoinRuntimeFilter; pub use physical_join_filter::PhysicalRuntimeFilter; pub use physical_join_filter::PhysicalRuntimeFilters; pub use physical_limit::Limit; +pub use physical_materialized_cte::MaterializedCTE; pub use physical_multi_table_insert::*; pub use physical_mutation::*; pub use physical_mutation_into_organize::MutationOrganize; @@ -107,7 +108,6 @@ pub use physical_replace_async_source::ReplaceAsyncSourcer; pub use physical_replace_deduplicate::*; pub use physical_replace_into::ReplaceInto; pub use physical_row_fetch::RowFetch; -pub use physical_materialized_cte::MaterializedCTE; pub use physical_sort::Sort; pub use physical_table_scan::TableScan; pub use physical_udf::Udf; diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index 901d1eed99764..b252b80d0e269 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -27,6 +27,7 @@ pub struct MaterializedCTE { pub stat_info: Option, pub left: Box, pub right: Box, + pub cte_name: String, } impl MaterializedCTE { From db16044ed4b484d0340bb92b72584900a30a2c55 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Sat, 21 Jun 2025 06:27:58 +0800 Subject: [PATCH 03/37] build pipeline --- .../builders/builder_cte_consumer.rs | 60 ++++++++++++++++ .../builders/builder_materialized_cte.rs | 6 +- .../service/src/pipelines/builders/mod.rs | 1 + .../service/src/pipelines/pipeline_builder.rs | 3 + .../processors/transforms/materialized_cte.rs | 69 ++++++++++++++++++- .../pipelines/processors/transforms/mod.rs | 1 + .../transform_recursive_cte_source.rs | 1 + src/query/sql/src/executor/format.rs | 16 +++++ src/query/sql/src/executor/physical_plan.rs | 14 +++- .../sql/src/executor/physical_plan_visitor.rs | 7 ++ .../sql/src/executor/physical_plans/mod.rs | 2 + .../physical_plans/physical_cte_consumer.rs | 35 ++++++++++ 12 files changed, 212 insertions(+), 3 deletions(-) create mode 100644 src/query/service/src/pipelines/builders/builder_cte_consumer.rs create mode 100644 src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs diff --git a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs new file mode 100644 index 0000000000000..adfe50a2b0fad --- /dev/null +++ b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs @@ -0,0 +1,60 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_sql::executor::physical_plans::CTEConsumer; +use databend_common_storages_fuse::TableContext; + +use crate::pipelines::processors::transforms::CTESource; +use crate::pipelines::PipelineBuilder; + +impl PipelineBuilder { + pub(crate) fn build_cte_consumer(&mut self, cte: &CTEConsumer) -> Result<()> { + let receiver = self + .cte_receivers + .get(&cte.cte_name) + .ok_or_else(|| { + ErrorCode::Internal(format!("CTE receiver not found for name: {}", cte.cte_name)) + })? + .clone(); + + let current_consumer_id = + *self + .next_cte_consumer_id + .get(&cte.cte_name) + .ok_or_else(|| { + ErrorCode::Internal(format!( + "CTE consumer id not found for name: {}", + cte.cte_name + )) + })?; + + self.next_cte_consumer_id + .insert(cte.cte_name.clone(), current_consumer_id + 1); + + self.main_pipeline.add_source( + |output_port| { + CTESource::create( + self.ctx.clone(), + output_port.clone(), + receiver.clone(), + current_consumer_id, + ) + }, + self.ctx.get_settings().get_max_threads()? as usize, + )?; + Ok(()) + } +} diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 385a07ccc8e44..02e42f3ee885f 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -12,9 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; + use databend_common_exception::Result; use databend_common_sql::executor::physical_plans::MaterializedCTE; +use crate::pipelines::processors::transforms::MaterializedCteData; use crate::pipelines::processors::transforms::MaterializedCteSink; use crate::pipelines::PipelineBuilder; use crate::sessions::QueryContext; @@ -32,8 +35,9 @@ impl PipelineBuilder { // build cte pipeline let mut build_res = sub_builder.finalize(&cte.left)?; build_res.main_pipeline.try_resize(1)?; - let (tx, rx) = tokio::sync::watch::channel(Default::default()); + let (tx, rx) = tokio::sync::watch::channel(Arc::new(MaterializedCteData::default())); self.cte_receivers.insert(cte.cte_name.clone(), rx); + self.next_cte_consumer_id.insert(cte.cte_name.clone(), 0); build_res .main_pipeline .add_sink(|input| MaterializedCteSink::create(input, tx.clone()))?; diff --git a/src/query/service/src/pipelines/builders/mod.rs b/src/query/service/src/pipelines/builders/mod.rs index a229fa18d008d..ba86e21fd6dc0 100644 --- a/src/query/service/src/pipelines/builders/mod.rs +++ b/src/query/service/src/pipelines/builders/mod.rs @@ -22,6 +22,7 @@ mod builder_commit; mod builder_compact; mod builder_copy_into_location; mod builder_copy_into_table; +mod builder_cte_consumer; mod builder_distributed_insert_select; mod builder_exchange; mod builder_fill_missing_columns; diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index c44fefd55cfbb..d7a593fe7c2cc 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -60,6 +60,7 @@ pub struct PipelineBuilder { pub contain_sink_processor: bool, pub cte_receivers: HashMap>>, + pub next_cte_consumer_id: HashMap, } impl PipelineBuilder { @@ -83,6 +84,7 @@ impl PipelineBuilder { contain_sink_processor: false, is_exchange_stack: vec![], cte_receivers: HashMap::new(), + next_cte_consumer_id: HashMap::new(), } } @@ -298,6 +300,7 @@ impl PipelineBuilder { )), PhysicalPlan::MaterializedCTE(cte) => self.build_materialized_cte(cte), + PhysicalPlan::CTEConsumer(cte_consumer) => self.build_cte_consumer(cte_consumer), }?; self.is_exchange_stack.pop(); diff --git a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs index 7d0d8e28c1018..092ae0ac94366 100644 --- a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs +++ b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs @@ -1,9 +1,12 @@ +use std::collections::HashMap; use std::sync::Arc; +use std::sync::Mutex; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_pipeline_core::processors::InputPort; +use databend_common_pipeline_core::processors::OutputPort; use databend_common_pipeline_core::processors::ProcessorPtr; // Copyright 2021 Datafuse Labs // @@ -20,6 +23,9 @@ use databend_common_pipeline_core::processors::ProcessorPtr; // limitations under the License. use databend_common_pipeline_sinks::Sink; use databend_common_pipeline_sinks::Sinker; +use databend_common_pipeline_sources::AsyncSource; +use databend_common_pipeline_sources::AsyncSourcer; +use tokio::sync::watch::Receiver; use tokio::sync::watch::Sender; pub struct MaterializedCteSink { @@ -30,11 +36,29 @@ pub struct MaterializedCteSink { #[derive(Default)] pub struct MaterializedCteData { blocks: Vec, + // consumer_id -> current_index + consumer_states: Arc>>, } impl MaterializedCteData { pub fn new(blocks: Vec) -> Self { - Self { blocks } + Self { + blocks, + consumer_states: Arc::new(Mutex::new(HashMap::new())), + } + } + + pub fn get_next_block(&self, consumer_id: usize) -> Option { + let mut states = self.consumer_states.lock().unwrap(); + let current_index = states.get(&consumer_id).copied().unwrap_or(0); + + if current_index < self.blocks.len() { + let block = self.blocks[current_index].clone(); + states.insert(consumer_id, current_index + 1); + Some(block) + } else { + None + } } } @@ -67,3 +91,46 @@ impl Sink for MaterializedCteSink { Ok(()) } } + +pub struct CTESource { + receiver: Receiver>, + data: Option>, + consumer_id: usize, +} + +impl CTESource { + pub fn create( + ctx: Arc, + output_port: Arc, + receiver: Receiver>, + consumer_id: usize, + ) -> Result { + AsyncSourcer::create(ctx, output_port, Self { + receiver, + data: None, + consumer_id, + }) + } +} + +#[async_trait::async_trait] +impl AsyncSource for CTESource { + const NAME: &'static str = "CTEConsumerSource"; + + #[async_backtrace::framed] + async fn generate(&mut self) -> Result> { + if self.data.is_none() { + self.receiver.changed().await.map_err(|_| { + ErrorCode::Internal("Failed to get data from receiver in CTEConsumerSource") + })?; + self.data = Some(self.receiver.borrow().clone()); + } + + if let Some(data) = &self.data { + if let Some(block) = data.get_next_block(self.consumer_id) { + return Ok(Some(block)); + } + } + Ok(None) + } +} diff --git a/src/query/service/src/pipelines/processors/transforms/mod.rs b/src/query/service/src/pipelines/processors/transforms/mod.rs index 25f8dea06f2c9..90aaaf365f3cf 100644 --- a/src/query/service/src/pipelines/processors/transforms/mod.rs +++ b/src/query/service/src/pipelines/processors/transforms/mod.rs @@ -47,6 +47,7 @@ mod window; pub use broadcast::BroadcastSinkProcessor; pub use broadcast::BroadcastSourceProcessor; pub use hash_join::*; +pub use materialized_cte::CTESource; pub use materialized_cte::MaterializedCteData; pub use materialized_cte::MaterializedCteSink; pub use transform_add_computed_columns::TransformAddComputedColumns; diff --git a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs index 14c353fd8c331..ee9b098b5ee57 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs @@ -356,6 +356,7 @@ async fn create_memory_table_for_cte_scan( | PhysicalPlan::ChunkMerge(_) | PhysicalPlan::ChunkCommitInsert(_) | PhysicalPlan::BroadcastSource(_) + | PhysicalPlan::CTEConsumer(_) | PhysicalPlan::BroadcastSink(_) => {} } Ok(()) diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index 95eaf484d59b7..c364da2bb00fb 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -537,6 +537,22 @@ fn to_format_tree( children, )) } + PhysicalPlan::CTEConsumer(plan) => { + let mut children = Vec::new(); + children.push(FormatTreeNode::new(format!( + "cte_name: {}", + plan.cte_name.clone() + ))); + children.push(FormatTreeNode::new(format!( + "cte_schema: [{}]", + format_output_columns(plan.cte_schema.clone(), metadata, false) + ))); + append_profile_info(&mut children, profs, plan.plan_id); + Ok(FormatTreeNode::with_children( + "CTEConsumer".to_string(), + children, + )) + } } } diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 2bbf03bb7ef70..7899b6aca9633 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -37,6 +37,7 @@ use crate::executor::physical_plans::AggregateExpand; use crate::executor::physical_plans::AggregateFinal; use crate::executor::physical_plans::AggregatePartial; use crate::executor::physical_plans::AsyncFunction; +use crate::executor::physical_plans::CTEConsumer; use crate::executor::physical_plans::CacheScan; use crate::executor::physical_plans::ChunkAppendData; use crate::executor::physical_plans::ChunkCastSchema; @@ -164,6 +165,7 @@ pub enum PhysicalPlan { BroadcastSink(BroadcastSink), MaterializedCTE(Box), + CTEConsumer(Box), } impl PhysicalPlan { @@ -431,6 +433,10 @@ impl PhysicalPlan { plan.left.adjust_plan_id(next_id); plan.right.adjust_plan_id(next_id); } + PhysicalPlan::CTEConsumer(plan) => { + plan.plan_id = *next_id; + *next_id += 1; + } } } @@ -490,6 +496,7 @@ impl PhysicalPlan { PhysicalPlan::BroadcastSource(v) => v.plan_id, PhysicalPlan::BroadcastSink(v) => v.plan_id, PhysicalPlan::MaterializedCTE(v) => v.plan_id, + PhysicalPlan::CTEConsumer(v) => v.plan_id, } } @@ -548,6 +555,7 @@ impl PhysicalPlan { PhysicalPlan::ChunkMerge(_) => todo!(), PhysicalPlan::ChunkCommitInsert(_) => todo!(), PhysicalPlan::MaterializedCTE(plan) => plan.output_schema(), + PhysicalPlan::CTEConsumer(plan) => plan.output_schema(), } } @@ -612,6 +620,7 @@ impl PhysicalPlan { PhysicalPlan::BroadcastSource(_) => "RuntimeFilterSource".to_string(), PhysicalPlan::BroadcastSink(_) => "RuntimeFilterSink".to_string(), PhysicalPlan::MaterializedCTE(_) => "MaterializedCTE".to_string(), + PhysicalPlan::CTEConsumer(_) => "CTEConsumer".to_string(), } } @@ -625,6 +634,7 @@ impl PhysicalPlan { | PhysicalPlan::ReplaceAsyncSourcer(_) | PhysicalPlan::Recluster(_) | PhysicalPlan::RecursiveCteScan(_) + | PhysicalPlan::CTEConsumer(_) | PhysicalPlan::BroadcastSource(_) => Box::new(std::iter::empty()), PhysicalPlan::HilbertPartition(plan) => Box::new(std::iter::once(plan.input.as_ref())), PhysicalPlan::Filter(plan) => Box::new(std::iter::once(plan.input.as_ref())), @@ -702,6 +712,7 @@ impl PhysicalPlan { | PhysicalPlan::ReplaceAsyncSourcer(_) | PhysicalPlan::Recluster(_) | PhysicalPlan::BroadcastSource(_) + | PhysicalPlan::CTEConsumer(_) | PhysicalPlan::RecursiveCteScan(_) => Box::new(std::iter::empty()), PhysicalPlan::HilbertPartition(plan) => Box::new(std::iter::once(plan.input.as_mut())), PhysicalPlan::Filter(plan) => Box::new(std::iter::once(plan.input.as_mut())), @@ -824,7 +835,8 @@ impl PhysicalPlan { | PhysicalPlan::ChunkCommitInsert(_) | PhysicalPlan::BroadcastSource(_) | PhysicalPlan::BroadcastSink(_) - | PhysicalPlan::MaterializedCTE(_) => None, + | PhysicalPlan::MaterializedCTE(_) + | PhysicalPlan::CTEConsumer(_) => None, } } diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index 3973665cc64b1..fa3ae41012517 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -29,6 +29,7 @@ use crate::executor::physical_plans::AggregateExpand; use crate::executor::physical_plans::AggregateFinal; use crate::executor::physical_plans::AggregatePartial; use crate::executor::physical_plans::AsyncFunction; +use crate::executor::physical_plans::CTEConsumer; use crate::executor::physical_plans::ChunkAppendData; use crate::executor::physical_plans::ChunkCastSchema; use crate::executor::physical_plans::ChunkCommitInsert; @@ -126,6 +127,7 @@ pub trait PhysicalPlanReplacer { PhysicalPlan::BroadcastSource(plan) => self.replace_runtime_filter_source(plan), PhysicalPlan::BroadcastSink(plan) => self.replace_runtime_filter_sink(plan), PhysicalPlan::MaterializedCTE(plan) => self.replace_materialized_cte(plan), + PhysicalPlan::CTEConsumer(plan) => self.replace_cte_consumer(plan), } } @@ -656,6 +658,10 @@ pub trait PhysicalPlanReplacer { cte_name: plan.cte_name.clone(), }))) } + + fn replace_cte_consumer(&mut self, plan: &CTEConsumer) -> Result { + Ok(PhysicalPlan::CTEConsumer(Box::new(plan.clone()))) + } } impl PhysicalPlan { @@ -679,6 +685,7 @@ impl PhysicalPlan { | PhysicalPlan::ExchangeSource(_) | PhysicalPlan::CompactSource(_) | PhysicalPlan::MutationSource(_) + | PhysicalPlan::CTEConsumer(_) | PhysicalPlan::BroadcastSource(_) => {} PhysicalPlan::Filter(plan) => { Self::traverse(&plan.input, pre_visit, visit, post_visit); diff --git a/src/query/sql/src/executor/physical_plans/mod.rs b/src/query/sql/src/executor/physical_plans/mod.rs index 6275ef8b46771..75b70be749b3b 100644 --- a/src/query/sql/src/executor/physical_plans/mod.rs +++ b/src/query/sql/src/executor/physical_plans/mod.rs @@ -61,6 +61,7 @@ mod physical_union_all; mod physical_window; mod physical_window_partition; +mod physical_cte_consumer; pub use common::*; pub use physical_add_stream_column::AddStreamColumn; pub use physical_aggregate_expand::AggregateExpand; @@ -78,6 +79,7 @@ pub use physical_compact_source::CompactSource; pub use physical_constant_table_scan::ConstantTableScan; pub use physical_copy_into_location::CopyIntoLocation; pub use physical_copy_into_table::*; +pub use physical_cte_consumer::*; pub use physical_distributed_insert_select::DistributedInsertSelect; pub use physical_eval_scalar::EvalScalar; pub use physical_exchange::Exchange; diff --git a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs new file mode 100644 index 0000000000000..e1e002f435d30 --- /dev/null +++ b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs @@ -0,0 +1,35 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_exception::Result; +use databend_common_expression::DataSchemaRef; + +use crate::executor::explain::PlanStatsInfo; + +/// This is a leaf operator that consumes the result of a materialized CTE. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct CTEConsumer { + // A unique id of operator in a `PhysicalPlan` tree, only used for display. + pub plan_id: u32, + // Only used for explain + pub stat_info: Option, + pub cte_name: String, + pub cte_schema: DataSchemaRef, +} + +impl CTEConsumer { + pub fn output_schema(&self) -> Result { + Ok(self.cte_schema.clone()) + } +} From 26bb0ab33ca6d19c79581b802cfef007df987773 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 23 Jun 2025 12:41:42 +0800 Subject: [PATCH 04/37] add operator --- .../sql/src/executor/physical_plan_builder.rs | 7 ++ .../physical_plans/physical_cte_consumer.rs | 17 ++++ .../physical_materialized_cte.rs | 21 +++++ src/query/sql/src/planner/binder/util.rs | 6 ++ .../src/planner/optimizer/ir/expr/s_expr.rs | 2 + .../sql/src/planner/optimizer/ir/format.rs | 2 + .../optimizer/optimizers/hyper_dp/dphyp.rs | 15 ++++ .../hyper_dp/dynamic_sample/dynamic_sample.rs | 2 + .../decorrelate/subquery_decorrelator.rs | 7 ++ .../join_rules/rule_semi_to_inner_join.rs | 2 + .../sql/src/planner/plans/cte_consumer.rs | 88 +++++++++++++++++++ .../sql/src/planner/plans/materialized_cte.rs | 42 +++++++++ src/query/sql/src/planner/plans/mod.rs | 4 + src/query/sql/src/planner/plans/operator.rs | 32 +++++++ 14 files changed, 247 insertions(+) create mode 100644 src/query/sql/src/planner/plans/cte_consumer.rs create mode 100644 src/query/sql/src/planner/plans/materialized_cte.rs diff --git a/src/query/sql/src/executor/physical_plan_builder.rs b/src/query/sql/src/executor/physical_plan_builder.rs index c50adc9f30db4..cc866a3f44b36 100644 --- a/src/query/sql/src/executor/physical_plan_builder.rs +++ b/src/query/sql/src/executor/physical_plan_builder.rs @@ -129,6 +129,13 @@ impl PhysicalPlanBuilder { self.build_mutation_source(mutation_source).await } RelOperator::CompactBlock(compact) => self.build_compact_block(compact).await, + RelOperator::MaterializedCTE(materialized_cte) => { + self.build_materialized_cte(s_expr, materialized_cte, stat_info) + .await + } + RelOperator::CTEConsumer(cte_consumer) => { + self.build_cte_consumer(cte_consumer, stat_info).await + } } } diff --git a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs index e1e002f435d30..819c5437ac5cb 100644 --- a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs +++ b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs @@ -16,6 +16,8 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::executor::explain::PlanStatsInfo; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanBuilder; /// This is a leaf operator that consumes the result of a materialized CTE. #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -33,3 +35,18 @@ impl CTEConsumer { Ok(self.cte_schema.clone()) } } + +impl PhysicalPlanBuilder { + pub(crate) async fn build_cte_consumer( + &mut self, + cte_consumer: &crate::plans::CTEConsumer, + stat_info: PlanStatsInfo, + ) -> Result { + Ok(PhysicalPlan::CTEConsumer(Box::new(CTEConsumer { + plan_id: 0, + stat_info: Some(stat_info), + cte_name: cte_consumer.cte_name.clone(), + cte_schema: Default::default(), + }))) + } +} diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index b252b80d0e269..6b42cba900749 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -17,6 +17,8 @@ use databend_common_expression::DataSchemaRef; use crate::executor::explain::PlanStatsInfo; use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanBuilder; +use crate::optimizer::ir::SExpr; /// This is a binary operator that executes its children in order (left to right), and returns the results of the right child #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -35,3 +37,22 @@ impl MaterializedCTE { self.right.output_schema() } } + +impl PhysicalPlanBuilder { + pub(crate) async fn build_materialized_cte( + &mut self, + s_expr: &SExpr, + materialized_cte: &crate::plans::MaterializedCTE, + stat_info: PlanStatsInfo, + ) -> Result { + let left_side = Box::new(self.build(s_expr.child(0)?, Default::default()).await?); + let right_side = Box::new(self.build(s_expr.child(1)?, Default::default()).await?); + Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { + plan_id: 0, + stat_info: Some(stat_info), + left: left_side, + right: right_side, + cte_name: materialized_cte.cte_name.clone(), + }))) + } +} diff --git a/src/query/sql/src/planner/binder/util.rs b/src/query/sql/src/planner/binder/util.rs index 039a42c2f806d..be6568d310337 100644 --- a/src/query/sql/src/planner/binder/util.rs +++ b/src/query/sql/src/planner/binder/util.rs @@ -53,6 +53,11 @@ impl Binder { self.count_r_cte_scan(expr.child(1)?, cte_scan_names, cte_types)?; } + RelOperator::MaterializedCTE(_) => { + self.count_r_cte_scan(expr.child(0)?, cte_scan_names, cte_types)?; + self.count_r_cte_scan(expr.child(1)?, cte_scan_names, cte_types)?; + } + RelOperator::ProjectSet(_) | RelOperator::AsyncFunction(_) | RelOperator::Udf(_) @@ -72,6 +77,7 @@ impl Binder { | RelOperator::DummyTableScan(_) | RelOperator::ConstantTableScan(_) | RelOperator::ExpressionScan(_) + | RelOperator::CTEConsumer(_) | RelOperator::CacheScan(_) => {} // Each recursive step in a recursive query generates new rows, and these rows are used for the next recursion. // Each step depends on the results of the previous step, so it's essential to ensure that the result set is built incrementally. diff --git a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs index ac784174ad191..f67a3416ea94c 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs @@ -374,6 +374,8 @@ impl SExpr { | RelOperator::CacheScan(_) | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) + | RelOperator::MaterializedCTE(_) + | RelOperator::CTEConsumer(_) | RelOperator::CompactBlock(_) => {} }; for child in &self.children { diff --git a/src/query/sql/src/planner/optimizer/ir/format.rs b/src/query/sql/src/planner/optimizer/ir/format.rs index f9613af6b35ef..5d6909f2cf374 100644 --- a/src/query/sql/src/planner/optimizer/ir/format.rs +++ b/src/query/sql/src/planner/optimizer/ir/format.rs @@ -80,6 +80,8 @@ fn display_rel_op(rel_op: &RelOperator) -> String { RelOperator::Mutation(_) => "MergeInto".to_string(), RelOperator::MutationSource(_) => "MutationSource".to_string(), RelOperator::CompactBlock(_) => "CompactBlock".to_string(), + RelOperator::MaterializedCTE(_) => "MaterializedCTE".to_string(), + RelOperator::CTEConsumer(_) => "CTEConsumer".to_string(), } } diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index 868b55f05e935..4d8c1d4daac5a 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -270,6 +270,18 @@ impl DPhpyOptimizer { Ok((new_s_expr, left_res.1 && right_res.1)) } + async fn process_materialized_cte_node( + &mut self, + s_expr: &SExpr, + ) -> Result<(Arc, bool)> { + let new_s_expr = self.new_children(s_expr).await?; + Ok((Arc::new(new_s_expr), true)) + } + + async fn process_cte_consumer_node(&mut self, s_expr: &SExpr) -> Result<(Arc, bool)> { + Ok((Arc::new(s_expr.clone()), true)) + } + /// Process a unary operator node async fn process_unary_node( &mut self, @@ -332,6 +344,9 @@ impl DPhpyOptimizer { RelOperator::Join(_) => self.process_join_node(s_expr, join_conditions).await, + RelOperator::MaterializedCTE(_) => self.process_materialized_cte_node(s_expr).await, + RelOperator::CTEConsumer(_) => self.process_cte_consumer_node(s_expr).await, + RelOperator::ProjectSet(_) | RelOperator::Aggregate(_) | RelOperator::Sort(_) diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs index eb645151c4230..cdc9071d13c84 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs @@ -92,6 +92,8 @@ pub async fn dynamic_sample( | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) | RelOperator::CompactBlock(_) + | RelOperator::MaterializedCTE(_) + | RelOperator::CTEConsumer(_) | RelOperator::MutationSource(_) => { s_expr.plan().derive_stats(&RelExpr::with_s_expr(s_expr)) } diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs index 079f80afcb9fe..a3f091cda6831 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs @@ -325,6 +325,12 @@ impl SubqueryDecorrelatorOptimizer { )) } + RelOperator::MaterializedCTE(_) => Ok(SExpr::create_binary( + s_expr.plan.clone(), + Arc::new(self.optimize_sync(s_expr.left_child())?), + Arc::new(self.optimize_sync(s_expr.right_child())?), + )), + RelOperator::DummyTableScan(_) | RelOperator::Scan(_) | RelOperator::ConstantTableScan(_) @@ -334,6 +340,7 @@ impl SubqueryDecorrelatorOptimizer { | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) | RelOperator::MutationSource(_) + | RelOperator::CTEConsumer(_) | RelOperator::CompactBlock(_) => Ok(s_expr.clone()), } } diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs index 46ebf4648860d..1cfa72a2ed63c 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs @@ -152,6 +152,8 @@ fn find_group_by_keys( | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) | RelOperator::MutationSource(_) + | RelOperator::MaterializedCTE(_) + | RelOperator::CTEConsumer(_) | RelOperator::CompactBlock(_) => {} } Ok(()) diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs new file mode 100644 index 0000000000000..af3450e008e78 --- /dev/null +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -0,0 +1,88 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::hash::Hash; +use std::hash::Hasher; + +use databend_common_expression::DataField; + +use crate::plans::Operator; +use crate::plans::RelOp; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CTEConsumer { + pub cte_name: String, + pub fields: Vec, +} + +impl CTEConsumer { + pub fn new(cte_name: String, fields: Vec) -> Self { + Self { cte_name, fields } + } + + // pub fn used_columns(&self) -> Result { + // let mut used_columns = ColumnSet::new(); + // for field in self.fields.iter() { + // used_columns.insert(field.name().parse()?); + // } + // Ok(used_columns) + // } +} + +impl Hash for CTEConsumer { + fn hash(&self, state: &mut H) { + self.cte_name.hash(state); + for field in self.fields.iter() { + field.name().hash(state); + } + } +} + +impl Operator for CTEConsumer { + fn rel_op(&self) -> RelOp { + RelOp::CTEConsumer + } + + // fn arity(&self) -> usize { + // 0 + // } + + // fn derive_relational_prop(&self, _rel_expr: &RelExpr) -> Result> { + // Ok(Arc::new(RelationalProperty { + // output_columns: self.used_columns()?, + // outer_columns: ColumnSet::new(), + // used_columns: self.used_columns()?, + // orderings: vec![], + // partition_orderings: None, + // })) + // } + + // fn derive_physical_prop(&self, _rel_expr: &RelExpr) -> Result { + // Ok(PhysicalProperty { + // distribution: Distribution::Serial, + // }) + // } + + // fn compute_required_prop_child( + // &self, + // _ctx: Arc, + // _rel_expr: &RelExpr, + // _child_index: usize, + // _required: &RequiredProperty, + // ) -> Result { + // Err(ErrorCode::Internal( + // "Cannot compute required property for CTEConsumer".to_string(), + // )) + // } +} diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs new file mode 100644 index 0000000000000..b7793e8bd3546 --- /dev/null +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -0,0 +1,42 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::hash::Hash; +use std::hash::Hasher; + +use crate::plans::Operator; +use crate::plans::RelOp; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MaterializedCTE { + pub cte_name: String, +} + +impl MaterializedCTE { + pub fn new(cte_name: String) -> Self { + Self { cte_name } + } +} + +impl Hash for MaterializedCTE { + fn hash(&self, state: &mut H) { + self.cte_name.hash(state); + } +} + +impl Operator for MaterializedCTE { + fn rel_op(&self) -> RelOp { + RelOp::MaterializedCTE + } +} diff --git a/src/query/sql/src/planner/plans/mod.rs b/src/query/sql/src/planner/plans/mod.rs index 2f324de8199d8..bcd237abc68a6 100644 --- a/src/query/sql/src/planner/plans/mod.rs +++ b/src/query/sql/src/planner/plans/mod.rs @@ -19,6 +19,7 @@ mod call; mod constant_table_scan; mod copy_into_location; mod copy_into_table; +mod cte_consumer; mod data_mask; mod ddl; mod dummy_table_scan; @@ -31,6 +32,7 @@ mod insert_multi_table; mod join; mod kill; mod limit; +mod materialized_cte; mod mutation; mod mutation_source; mod operator; @@ -61,6 +63,7 @@ pub use call::CallPlan; pub use constant_table_scan::ConstantTableScan; pub use copy_into_location::*; pub use copy_into_table::*; +pub use cte_consumer::*; pub use data_mask::*; pub use ddl::*; pub use dummy_table_scan::DummyTableScan; @@ -73,6 +76,7 @@ pub use insert_multi_table::*; pub use join::*; pub use kill::KillPlan; pub use limit::*; +pub use materialized_cte::*; pub use mutation::MatchedEvaluator; pub use mutation::Mutation; pub use mutation::UnmatchedEvaluator; diff --git a/src/query/sql/src/planner/plans/operator.rs b/src/query/sql/src/planner/plans/operator.rs index 3d3a67518fed0..d31852361cfd4 100644 --- a/src/query/sql/src/planner/plans/operator.rs +++ b/src/query/sql/src/planner/plans/operator.rs @@ -29,6 +29,7 @@ use crate::optimizer::ir::StatInfo; use crate::plans::r_cte_scan::RecursiveCteScan; use crate::plans::Aggregate; use crate::plans::AsyncFunction; +use crate::plans::CTEConsumer; use crate::plans::CacheScan; use crate::plans::ConstantTableScan; use crate::plans::DummyTableScan; @@ -38,6 +39,7 @@ use crate::plans::ExpressionScan; use crate::plans::Filter; use crate::plans::Join; use crate::plans::Limit; +use crate::plans::MaterializedCTE; use crate::plans::Mutation; use crate::plans::OptimizeCompactBlock; use crate::plans::ProjectSet; @@ -119,6 +121,8 @@ pub enum RelOp { MergeInto, CompactBlock, MutationSource, + MaterializedCTE, + CTEConsumer, // Pattern Pattern, @@ -155,6 +159,8 @@ pub enum RelOperator { Mutation(Mutation), CompactBlock(OptimizeCompactBlock), MutationSource(MutationSource), + MaterializedCTE(MaterializedCTE), + CTEConsumer(CTEConsumer), } impl RelOperator { @@ -172,6 +178,8 @@ impl RelOperator { | RelOperator::AsyncFunction(_) | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) + | RelOperator::MaterializedCTE(_) + | RelOperator::CTEConsumer(_) | RelOperator::CompactBlock(_) => false, RelOperator::Join(op) => op.has_subquery(), RelOperator::EvalScalar(op) => op.items.iter().any(|expr| expr.scalar.has_subquery()), @@ -219,6 +227,8 @@ impl RelOperator { | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) | RelOperator::CompactBlock(_) + | RelOperator::MaterializedCTE(_) + | RelOperator::CTEConsumer(_) | RelOperator::MutationSource(_) => (), RelOperator::Join(op) => { for condition in &op.equi_conditions { @@ -299,6 +309,8 @@ impl Operator for RelOperator { RelOperator::Mutation(rel_op) => rel_op.rel_op(), RelOperator::CompactBlock(rel_op) => rel_op.rel_op(), RelOperator::MutationSource(rel_op) => rel_op.rel_op(), + RelOperator::MaterializedCTE(rel_op) => rel_op.rel_op(), + RelOperator::CTEConsumer(rel_op) => rel_op.rel_op(), } } @@ -325,6 +337,8 @@ impl Operator for RelOperator { RelOperator::Mutation(rel_op) => rel_op.arity(), RelOperator::CompactBlock(rel_op) => rel_op.arity(), RelOperator::MutationSource(rel_op) => rel_op.arity(), + RelOperator::MaterializedCTE(rel_op) => rel_op.arity(), + RelOperator::CTEConsumer(rel_op) => rel_op.arity(), } } @@ -351,6 +365,8 @@ impl Operator for RelOperator { RelOperator::Mutation(rel_op) => rel_op.derive_relational_prop(rel_expr), RelOperator::CompactBlock(rel_op) => rel_op.derive_relational_prop(rel_expr), RelOperator::MutationSource(rel_op) => rel_op.derive_relational_prop(rel_expr), + RelOperator::MaterializedCTE(rel_op) => rel_op.derive_relational_prop(rel_expr), + RelOperator::CTEConsumer(rel_op) => rel_op.derive_relational_prop(rel_expr), } } @@ -377,6 +393,8 @@ impl Operator for RelOperator { RelOperator::Mutation(rel_op) => rel_op.derive_physical_prop(rel_expr), RelOperator::CompactBlock(rel_op) => rel_op.derive_physical_prop(rel_expr), RelOperator::MutationSource(rel_op) => rel_op.derive_physical_prop(rel_expr), + RelOperator::MaterializedCTE(rel_op) => rel_op.derive_physical_prop(rel_expr), + RelOperator::CTEConsumer(rel_op) => rel_op.derive_physical_prop(rel_expr), } } @@ -403,6 +421,8 @@ impl Operator for RelOperator { RelOperator::Mutation(rel_op) => rel_op.derive_stats(rel_expr), RelOperator::CompactBlock(rel_op) => rel_op.derive_stats(rel_expr), RelOperator::MutationSource(rel_op) => rel_op.derive_stats(rel_expr), + RelOperator::MaterializedCTE(rel_op) => rel_op.derive_stats(rel_expr), + RelOperator::CTEConsumer(rel_op) => rel_op.derive_stats(rel_expr), } } @@ -477,6 +497,12 @@ impl Operator for RelOperator { RelOperator::MutationSource(rel_op) => { rel_op.compute_required_prop_child(ctx, rel_expr, child_index, required) } + RelOperator::MaterializedCTE(rel_op) => { + rel_op.compute_required_prop_child(ctx, rel_expr, child_index, required) + } + RelOperator::CTEConsumer(rel_op) => { + rel_op.compute_required_prop_child(ctx, rel_expr, child_index, required) + } } } @@ -550,6 +576,12 @@ impl Operator for RelOperator { RelOperator::MutationSource(rel_op) => { rel_op.compute_required_prop_children(ctx, rel_expr, required) } + RelOperator::MaterializedCTE(rel_op) => { + rel_op.compute_required_prop_children(ctx, rel_expr, required) + } + RelOperator::CTEConsumer(rel_op) => { + rel_op.compute_required_prop_children(ctx, rel_expr, required) + } } } } From 7e986a974c459c428c14eec042452f31b360530c Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 23 Jun 2025 15:39:41 +0800 Subject: [PATCH 05/37] remove m cte temp table --- src/query/catalog/src/table_context.rs | 4 - .../src/interpreters/hook/compact_hook.rs | 2 - .../src/interpreters/hook/refresh_hook.rs | 3 - .../src/interpreters/hook/vacuum_hook.rs | 8 - .../service/src/interpreters/interpreter.rs | 2 - .../interpreters/interpreter_table_create.rs | 2 - .../interpreter_table_recluster.rs | 2 - src/query/service/src/interpreters/mod.rs | 1 - .../src/servers/http/v1/query/http_query.rs | 3 - src/query/service/src/sessions/query_ctx.rs | 45 ----- .../tests/it/sql/exec/get_table_bind_test.rs | 8 - .../it/storages/fuse/operations/commit.rs | 9 +- .../sql/src/planner/binder/bind_query/bind.rs | 170 ------------------ 13 files changed, 1 insertion(+), 258 deletions(-) diff --git a/src/query/catalog/src/table_context.rs b/src/query/catalog/src/table_context.rs index d4dc75c7d0215..42b9141b5e469 100644 --- a/src/query/catalog/src/table_context.rs +++ b/src/query/catalog/src/table_context.rs @@ -403,10 +403,6 @@ pub trait TableContext: Send + Sync { fn is_temp_table(&self, catalog_name: &str, database_name: &str, table_name: &str) -> bool; fn get_shared_settings(&self) -> Arc; - fn add_m_cte_temp_table(&self, database_name: &str, table_name: &str); - - async fn drop_m_cte_temp_table(&self) -> Result<()>; - fn add_streams_ref(&self, _catalog: &str, _database: &str, _stream: &str, _consume: bool) { unimplemented!() } diff --git a/src/query/service/src/interpreters/hook/compact_hook.rs b/src/query/service/src/interpreters/hook/compact_hook.rs index 0029a2b0ff0b1..e1da58ed2147d 100644 --- a/src/query/service/src/interpreters/hook/compact_hook.rs +++ b/src/query/service/src/interpreters/hook/compact_hook.rs @@ -33,7 +33,6 @@ use log::info; use crate::interpreters::common::metrics_inc_compact_hook_compact_time_ms; use crate::interpreters::common::metrics_inc_compact_hook_main_operation_time_ms; -use crate::interpreters::hook::vacuum_hook::hook_clear_m_cte_temp_table; use crate::interpreters::hook::vacuum_hook::hook_disk_temp_dir; use crate::interpreters::hook::vacuum_hook::hook_vacuum_temp_files; use crate::interpreters::Interpreter; @@ -183,7 +182,6 @@ async fn compact_table( let query_ctx = ctx.clone(); build_res.main_pipeline.set_on_finished(always_callback( move |_info: &ExecutionInfo| { - hook_clear_m_cte_temp_table(&query_ctx)?; hook_vacuum_temp_files(&query_ctx)?; hook_disk_temp_dir(&query_ctx)?; Ok(()) diff --git a/src/query/service/src/interpreters/hook/refresh_hook.rs b/src/query/service/src/interpreters/hook/refresh_hook.rs index 3589c79459a59..64a3bc3920f13 100644 --- a/src/query/service/src/interpreters/hook/refresh_hook.rs +++ b/src/query/service/src/interpreters/hook/refresh_hook.rs @@ -38,7 +38,6 @@ use databend_storages_common_table_meta::meta::Location; use log::info; use parking_lot::RwLock; -use crate::interpreters::hook::vacuum_hook::hook_clear_m_cte_temp_table; use crate::interpreters::hook::vacuum_hook::hook_disk_temp_dir; use crate::interpreters::hook::vacuum_hook::hook_vacuum_temp_files; use crate::interpreters::Interpreter; @@ -125,7 +124,6 @@ async fn do_refresh(ctx: Arc, desc: RefreshDesc) -> Result<()> { let query_ctx = ctx_cloned.clone(); build_res.main_pipeline.set_on_finished(always_callback( move |_: &ExecutionInfo| { - hook_clear_m_cte_temp_table(&query_ctx)?; hook_vacuum_temp_files(&query_ctx)?; hook_disk_temp_dir(&query_ctx)?; Ok(()) @@ -162,7 +160,6 @@ async fn do_refresh(ctx: Arc, desc: RefreshDesc) -> Result<()> { let query_ctx = ctx_cloned.clone(); build_res.main_pipeline.set_on_finished(always_callback( move |_info: &ExecutionInfo| { - hook_clear_m_cte_temp_table(&query_ctx)?; hook_vacuum_temp_files(&query_ctx)?; hook_disk_temp_dir(&query_ctx)?; Ok(()) diff --git a/src/query/service/src/interpreters/hook/vacuum_hook.rs b/src/query/service/src/interpreters/hook/vacuum_hook.rs index b1f0aaa977ecb..93b886094e4c1 100644 --- a/src/query/service/src/interpreters/hook/vacuum_hook.rs +++ b/src/query/service/src/interpreters/hook/vacuum_hook.rs @@ -102,11 +102,3 @@ pub fn hook_disk_temp_dir(query_ctx: &Arc) -> Result<()> { Ok(()) } - -pub fn hook_clear_m_cte_temp_table(query_ctx: &Arc) -> Result<()> { - let _ = GlobalIORuntime::instance().block_on(async move { - query_ctx.drop_m_cte_temp_table().await?; - Ok(()) - }); - Ok(()) -} diff --git a/src/query/service/src/interpreters/interpreter.rs b/src/query/service/src/interpreters/interpreter.rs index 1d747ba85ad4b..c3c0a98c79113 100644 --- a/src/query/service/src/interpreters/interpreter.rs +++ b/src/query/service/src/interpreters/interpreter.rs @@ -49,7 +49,6 @@ use log::info; use md5::Digest; use md5::Md5; -use super::hook::vacuum_hook::hook_clear_m_cte_temp_table; use super::hook::vacuum_hook::hook_disk_temp_dir; use super::hook::vacuum_hook::hook_vacuum_temp_files; use super::InterpreterMetrics; @@ -363,7 +362,6 @@ pub fn on_execution_finished(info: &ExecutionInfo, query_ctx: Arc) ); } - hook_clear_m_cte_temp_table(&query_ctx)?; hook_vacuum_temp_files(&query_ctx)?; hook_disk_temp_dir(&query_ctx)?; diff --git a/src/query/service/src/interpreters/interpreter_table_create.rs b/src/query/service/src/interpreters/interpreter_table_create.rs index 0886f95104fef..920ae8e6d9290 100644 --- a/src/query/service/src/interpreters/interpreter_table_create.rs +++ b/src/query/service/src/interpreters/interpreter_table_create.rs @@ -74,7 +74,6 @@ use crate::interpreters::common::table_option_validation::is_valid_data_retentio use crate::interpreters::common::table_option_validation::is_valid_option_of_type; use crate::interpreters::common::table_option_validation::is_valid_random_seed; use crate::interpreters::common::table_option_validation::is_valid_row_per_block; -use crate::interpreters::hook::vacuum_hook::hook_clear_m_cte_temp_table; use crate::interpreters::hook::vacuum_hook::hook_disk_temp_dir; use crate::interpreters::hook::vacuum_hook::hook_vacuum_temp_files; use crate::interpreters::InsertInterpreter; @@ -280,7 +279,6 @@ impl CreateTableInterpreter { pipeline .main_pipeline .set_on_finished(always_callback(move |_: &ExecutionInfo| { - hook_clear_m_cte_temp_table(&query_ctx)?; hook_vacuum_temp_files(&query_ctx)?; hook_disk_temp_dir(&query_ctx)?; Ok(()) diff --git a/src/query/service/src/interpreters/interpreter_table_recluster.rs b/src/query/service/src/interpreters/interpreter_table_recluster.rs index f3c53597b06d7..3c27d2ba621f9 100644 --- a/src/query/service/src/interpreters/interpreter_table_recluster.rs +++ b/src/query/service/src/interpreters/interpreter_table_recluster.rs @@ -68,7 +68,6 @@ use derive_visitor::DriveMut; use log::error; use log::warn; -use crate::interpreters::hook::vacuum_hook::hook_clear_m_cte_temp_table; use crate::interpreters::hook::vacuum_hook::hook_disk_temp_dir; use crate::interpreters::hook::vacuum_hook::hook_vacuum_temp_files; use crate::interpreters::interpreter_insert_multi_table::scalar_expr_to_remote_expr; @@ -247,7 +246,6 @@ impl ReclusterTableInterpreter { ctx.evict_table_from_cache(&catalog, &database, &table)?; ctx.unload_spill_meta(); - hook_clear_m_cte_temp_table(&ctx)?; hook_vacuum_temp_files(&ctx)?; hook_disk_temp_dir(&ctx)?; match &info.res { diff --git a/src/query/service/src/interpreters/mod.rs b/src/query/service/src/interpreters/mod.rs index 22b7783f332a1..1587886a868b7 100644 --- a/src/query/service/src/interpreters/mod.rs +++ b/src/query/service/src/interpreters/mod.rs @@ -173,7 +173,6 @@ mod util; pub use access::ManagementModeAccess; pub use common::InterpreterQueryLog; -pub use hook::vacuum_hook::hook_clear_m_cte_temp_table; pub use hook::HookOperator; pub use interpreter::interpreter_plan_sql; pub use interpreter::Interpreter; diff --git a/src/query/service/src/servers/http/v1/query/http_query.rs b/src/query/service/src/servers/http/v1/query/http_query.rs index 147f38520d77a..e6f1888c5f2ad 100644 --- a/src/query/service/src/servers/http/v1/query/http_query.rs +++ b/src/query/service/src/servers/http/v1/query/http_query.rs @@ -791,9 +791,6 @@ impl HttpQuery { .with_context(|| "failed to start query") .flatten() { - crate::interpreters::hook_clear_m_cte_temp_table(&query_context) - .inspect_err(|e| warn!("clear_m_cte_temp_table fail: {e}")) - .ok(); let state = ExecuteStopped { stats: Progresses::default(), schema: vec![], diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index ef41cfd4048d9..cbd2898980ba6 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -89,7 +89,6 @@ use databend_common_meta_app::principal::UserPrivilegeType; use databend_common_meta_app::principal::COPY_MAX_FILES_COMMIT_MSG; use databend_common_meta_app::principal::COPY_MAX_FILES_PER_COMMIT; use databend_common_meta_app::schema::CatalogType; -use databend_common_meta_app::schema::DropTableByIdReq; use databend_common_meta_app::schema::GetTableCopiedFileReq; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::storage::StorageParams; @@ -122,13 +121,11 @@ use databend_common_users::GrantObjectVisibilityChecker; use databend_common_users::UserApiProvider; use databend_common_version::DATABEND_COMMIT_VERSION; use databend_common_version::DATABEND_ENTERPRISE_LICENSE_EMBEDDED; -use databend_storages_common_session::drop_table_by_id; use databend_storages_common_session::SessionState; use databend_storages_common_session::TxnManagerRef; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; -use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; use jiff::tz::TimeZone; use jiff::Zoned; use log::debug; @@ -168,9 +165,6 @@ pub struct QueryContext { fragment_id: Arc, // Used by synchronized generate aggregating indexes when new data written. written_segment_locs: Arc>>, - // Temp table for materialized CTE, first string is the database_name, second string is the table_name - // All temp tables' catalog is `CATALOG_DEFAULT`, so we don't need to store it. - m_cte_temp_table: Arc>>, } impl QueryContext { @@ -196,7 +190,6 @@ impl QueryContext { fragment_id: Arc::new(AtomicUsize::new(0)), written_segment_locs: Default::default(), block_threshold: Default::default(), - m_cte_temp_table: Default::default(), }) } @@ -1827,44 +1820,6 @@ impl TableContext for QueryContext { .is_temp_table(database_name, table_name) } - fn add_m_cte_temp_table(&self, database_name: &str, table_name: &str) { - self.m_cte_temp_table - .write() - .push((database_name.to_string(), table_name.to_string())); - } - - async fn drop_m_cte_temp_table(&self) -> Result<()> { - let temp_tbl_mgr = self.shared.session.session_ctx.temp_tbl_mgr(); - let m_cte_temp_table = self.m_cte_temp_table.read().clone(); - let tenant = self.get_tenant(); - for (db_name, table_name) in m_cte_temp_table.iter() { - let table = self.get_table(CATALOG_DEFAULT, db_name, table_name).await?; - let db = self - .get_catalog(CATALOG_DEFAULT) - .await? - .get_database(&tenant, db_name) - .await?; - let drop_table_req = DropTableByIdReq { - if_exists: true, - tenant: tenant.clone(), - tb_id: table.get_table_info().ident.table_id, - table_name: table_name.to_string(), - db_id: db.get_db_info().database_id.db_id, - db_name: db.name().to_string(), - engine: table.engine().to_string(), - temp_prefix: table - .options() - .get(OPT_KEY_TEMP_PREFIX) - .cloned() - .unwrap_or_default(), - }; - drop_table_by_id(temp_tbl_mgr.clone(), drop_table_req).await?; - } - let mut m_cte_temp_table = self.m_cte_temp_table.write(); - m_cte_temp_table.clear(); - Ok(()) - } - fn add_streams_ref(&self, catalog: &str, database: &str, stream: &str, consume: bool) { let mut streams = self.shared.streams_refs.write(); let stream_key = ( diff --git a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs index 978f0fc79b8d8..79cbad0ff12ee 100644 --- a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs +++ b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs @@ -995,14 +995,6 @@ impl TableContext for CtxDelegation { false } - fn add_m_cte_temp_table(&self, _database_name: &str, _table_name: &str) { - todo!() - } - - async fn drop_m_cte_temp_table(&self) -> Result<()> { - todo!() - } - fn set_cluster(&self, _: Arc) { todo!() } diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index 1fd9bdcfa23b9..7db1e682e7d3c 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -869,18 +869,11 @@ impl TableContext for CtxDelegation { fn is_temp_table(&self, _catalog_name: &str, _database_name: &str, _table_name: &str) -> bool { false } - fn add_m_cte_temp_table(&self, _database_name: &str, _table_name: &str) { - todo!() - } - - async fn drop_m_cte_temp_table(&self) -> Result<()> { - todo!() - } fn set_cluster(&self, _: Arc) { todo!() } - + async fn get_warehouse_cluster(&self) -> Result> { todo!() } diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 46867955cb746..72ee1d755812f 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -15,28 +15,15 @@ use std::collections::HashMap; use std::sync::Arc; -use databend_common_ast::ast::ColumnDefinition; -use databend_common_ast::ast::CreateOption; -use databend_common_ast::ast::CreateTableSource; -use databend_common_ast::ast::CreateTableStmt; -use databend_common_ast::ast::Engine; use databend_common_ast::ast::Expr; -use databend_common_ast::ast::Identifier; use databend_common_ast::ast::Query; use databend_common_ast::ast::SetExpr; use databend_common_ast::ast::TableReference; -use databend_common_ast::ast::TableType; use databend_common_ast::ast::With; -use databend_common_ast::ast::CTE; -use databend_common_ast::Span; -use databend_common_catalog::catalog::CATALOG_DEFAULT; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::convert_to_type_name; use derive_visitor::Drive; -use derive_visitor::DriveMut; use derive_visitor::Visitor; -use derive_visitor::VisitorMut; use crate::binder::CteInfo; use crate::normalize_identifier; @@ -163,10 +150,6 @@ impl Binder { columns: vec![], materialized: cte.materialized, }; - // If the CTE is materialized, we'll construct a temp table for it. - if cte.materialized { - self.m_cte_to_temp_table(cte, idx, with.clone())?; - } bind_context .cte_context .cte_map @@ -243,157 +226,4 @@ impl Binder { Arc::new(child), )) } - - fn m_cte_to_temp_table(&mut self, cte: &CTE, cte_index: usize, mut with: With) -> Result<()> { - let engine = if self.ctx.get_settings().get_persist_materialized_cte()? { - Engine::Fuse - } else { - Engine::Memory - }; - let query_id = self.ctx.get_id(); - let database = self.ctx.get_current_database(); - let mut table_identifier = cte.alias.name.clone(); - table_identifier.name = format!("{}${}", table_identifier.name, query_id.replace("-", "")); - let table_name = normalize_identifier(&table_identifier, &self.name_resolution_ctx).name; - self.m_cte_table_name.insert( - normalize_identifier(&cte.alias.name, &self.name_resolution_ctx).name, - table_name.clone(), - ); - if self - .ctx - .is_temp_table(CATALOG_DEFAULT, &database, &table_name) - { - return Err(ErrorCode::Internal(format!( - "Temporary table {:?} already exists in current session, please change the materialized CTE name", - table_name - ))); - } - - let mut expr_replacer = TableNameReplacer::new( - database.clone(), - self.m_cte_table_name.clone(), - self.name_resolution_ctx.clone(), - ); - let mut as_query = cte.query.clone(); - with.ctes.truncate(cte_index); - with.ctes.retain(|cte| !cte.materialized); - as_query.with = if !with.ctes.is_empty() { - Some(with) - } else { - None - }; - as_query.drive_mut(&mut expr_replacer); - - let source = if cte.alias.columns.is_empty() { - None - } else { - let mut bind_context = BindContext::new(); - let (_, bind_context) = self.bind_query(&mut bind_context, &as_query)?; - let columns = &bind_context.columns; - if columns.len() != cte.alias.columns.len() { - return Err(ErrorCode::Internal("Number of columns does not match")); - } - Some(CreateTableSource::Columns( - columns - .iter() - .zip(cte.alias.columns.iter()) - .map(|(column, ident)| { - let data_type = convert_to_type_name(&column.data_type); - ColumnDefinition { - name: ident.clone(), - data_type, - expr: None, - comment: None, - } - }) - .collect(), - None, - )) - }; - - let catalog = self.ctx.get_current_catalog(); - let create_table_stmt = CreateTableStmt { - create_option: CreateOption::Create, - catalog: Some(Identifier::from_name(Span::None, catalog.clone())), - database: Some(Identifier::from_name(Span::None, database.clone())), - table: table_identifier, - source, - engine: Some(engine), - uri_location: None, - cluster_by: None, - table_options: Default::default(), - iceberg_table_partition: None, - table_properties: Default::default(), - as_query: Some(as_query), - table_type: TableType::Temporary, - }; - - let create_table_sql = create_table_stmt.to_string(); - log::info!("[CTE]create_table_sql: {create_table_sql}"); - if let Some(subquery_executor) = &self.subquery_executor { - let _ = databend_common_base::runtime::block_on(async move { - subquery_executor - .execute_query_with_sql_string(&create_table_sql) - .await - })?; - } else { - return Err(ErrorCode::Internal("Binder's Subquery executor is not set")); - }; - - self.ctx.add_m_cte_temp_table(&database, &table_name); - - self.ctx - .evict_table_from_cache(&catalog, &database, &table_name) - } -} - -#[derive(VisitorMut)] -#[visitor(TableReference(enter), Expr(enter))] -pub struct TableNameReplacer { - database: String, - new_name: HashMap, - name_resolution_ctx: NameResolutionContext, -} - -impl TableNameReplacer { - pub fn new( - database: String, - new_name: HashMap, - name_resolution_ctx: NameResolutionContext, - ) -> Self { - Self { - database, - new_name, - name_resolution_ctx, - } - } - - fn replace_identifier(&mut self, identifier: &mut Identifier) { - let name = normalize_identifier(identifier, &self.name_resolution_ctx).name; - if let Some(new_name) = self.new_name.get(&name) { - identifier.name = new_name.clone(); - } - } - - fn enter_table_reference(&mut self, table_reference: &mut TableReference) { - if let TableReference::Table { - database, table, .. - } = table_reference - { - if database.is_none() || database.as_ref().unwrap().name == self.database { - self.replace_identifier(table); - } - } - } - - fn enter_expr(&mut self, expr: &mut Expr) { - if let Expr::ColumnRef { column, .. } = expr { - if column.database.is_none() || column.database.as_ref().unwrap().name == self.database - { - if let Some(table_identifier) = &mut column.table { - self.replace_identifier(table_identifier); - } - } - } - } } From e4ee842503209327e7a6ec369223518b74e3de5c Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 23 Jun 2025 20:49:53 +0800 Subject: [PATCH 06/37] bind --- .../it/storages/fuse/operations/commit.rs | 2 +- .../sql/src/planner/binder/bind_context.rs | 1 + .../sql/src/planner/binder/bind_query/bind.rs | 42 +++++++++++++++++++ .../binder/bind_table_reference/bind_table.rs | 19 ++++----- .../bind_table_function.rs | 2 - src/query/sql/src/planner/binder/table.rs | 1 - src/query/sql/src/planner/dataframe.rs | 1 - .../planner/expression/expression_parser.rs | 1 - .../sql/src/planner/metadata/metadata.rs | 11 ----- .../sql/src/planner/plans/cte_consumer.rs | 37 +++++----------- .../sql/src/planner/plans/materialized_cte.rs | 9 +--- src/query/sql/src/planner/plans/operator.rs | 20 +++++++++ 12 files changed, 83 insertions(+), 63 deletions(-) diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index 7db1e682e7d3c..4bbb7fdf2a47b 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -873,7 +873,7 @@ impl TableContext for CtxDelegation { fn set_cluster(&self, _: Arc) { todo!() } - + async fn get_warehouse_cluster(&self) -> Result> { todo!() } diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 2cf026e08983e..3f74fe384b8a6 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -160,6 +160,7 @@ pub struct CteContext { /// If the `BindContext` is created from a CTE, record the cte name pub cte_name: Option, pub cte_map: Box>, + pub is_binding_materialized_cte: bool, } impl CteContext { diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 72ee1d755812f..83d3b0a6e16b2 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -31,7 +31,9 @@ use crate::optimizer::ir::SExpr; use crate::planner::binder::scalar::ScalarBinder; use crate::planner::binder::BindContext; use crate::planner::binder::Binder; +use crate::planner::binder::CteContext; use crate::plans::BoundColumnRef; +use crate::plans::MaterializedCTE; use crate::plans::ScalarExpr; use crate::plans::Sort; use crate::plans::SortItem; @@ -85,6 +87,10 @@ impl Binder { // Bind limit. s_expr = self.bind_query_limit(query, s_expr, limit, offset); + if let Some(with) = &with { + s_expr = self.bind_materialized_cte(with, s_expr, bind_context.cte_context.clone())?; + } + Ok((s_expr, bind_context)) } @@ -226,4 +232,40 @@ impl Binder { Arc::new(child), )) } + + fn bind_materialized_cte( + &mut self, + with: &With, + main_query_expr: SExpr, + mut cte_context: CteContext, + ) -> Result { + let mut current_expr = main_query_expr; + cte_context.is_binding_materialized_cte = true; + + for cte in with.ctes.iter().rev() { + if cte.materialized { + let cte_name = self.normalize_identifier(&cte.alias.name).name; + + // Create a new bind context for the CTE definition + let mut cte_bind_context = BindContext { + cte_context: cte_context.clone(), + ..Default::default() + }; + + // Bind the CTE definition + let (cte_definition_expr, _) = + self.bind_query(&mut cte_bind_context, &cte.query)?; + + // Create the MaterializedCTE operator + let materialized_cte = MaterializedCTE::new(cte_name); + current_expr = SExpr::create_binary( + Arc::new(materialized_cte.into()), + Arc::new(cte_definition_expr), + Arc::new(current_expr), + ); + } + } + + Ok(current_expr) + } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 7848170697095..685cbf6f6eaf5 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; + use databend_common_ast::ast::Identifier; use databend_common_ast::ast::SampleConfig; use databend_common_ast::ast::Statement; @@ -33,6 +35,8 @@ use databend_storages_common_table_meta::table::get_change_type; use crate::binder::util::TableIdentifier; use crate::binder::Binder; use crate::optimizer::ir::SExpr; +use crate::plans::CTEConsumer; +use crate::plans::RelOperator; use crate::BindContext; impl Binder { @@ -69,12 +73,13 @@ impl Binder { (false, None, String::new()) }; - // Check and bind common table expression - let mut cte_suffix_name = None; let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { if cte_info.materialized { - cte_suffix_name = Some(self.ctx.get_id().replace("-", "")); + let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { + cte_name: table_name, + }))); + return Ok((s_expr, bind_context.clone())); } else { if self .metadata @@ -104,11 +109,6 @@ impl Binder { // Resolve table with catalog let table_meta = { - let table_name = if let Some(cte_suffix_name) = cte_suffix_name.as_ref() { - format!("{}${}", &table_name, cte_suffix_name) - } else { - table_name.clone() - }; match self.resolve_data_source( catalog.as_str(), database.as_str(), @@ -161,7 +161,6 @@ impl Binder { bind_context.view_info.is_some(), bind_context.planning_agg_index, false, - None, false, ); let (s_expr, mut bind_context) = self.bind_base_table( @@ -247,7 +246,6 @@ impl Binder { false, false, false, - None, bind_context.allow_virtual_column, ); let (s_expr, mut new_bind_context) = @@ -280,7 +278,6 @@ impl Binder { bind_context.view_info.is_some(), bind_context.planning_agg_index, false, - cte_suffix_name, bind_context.allow_virtual_column, ); diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs index 3cfe23223548f..31c9b8bcbd0df 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs @@ -151,7 +151,6 @@ impl Binder { false, false, false, - None, false, ); @@ -214,7 +213,6 @@ impl Binder { false, false, false, - None, false, ); diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index 23ff11114564a..e9785f40f5922 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -142,7 +142,6 @@ impl Binder { false, false, true, - None, bind_context.allow_virtual_column, ); diff --git a/src/query/sql/src/planner/dataframe.rs b/src/query/sql/src/planner/dataframe.rs index 277a010eace81..98ac53efc657d 100644 --- a/src/query/sql/src/planner/dataframe.rs +++ b/src/query/sql/src/planner/dataframe.rs @@ -101,7 +101,6 @@ impl Dataframe { false, false, false, - None, false, ); diff --git a/src/query/sql/src/planner/expression/expression_parser.rs b/src/query/sql/src/planner/expression/expression_parser.rs index 52ed0cc99a6fb..8937e880ec55f 100644 --- a/src/query/sql/src/planner/expression/expression_parser.rs +++ b/src/query/sql/src/planner/expression/expression_parser.rs @@ -63,7 +63,6 @@ pub fn bind_table(table_meta: Arc) -> Result<(BindContext, MetadataRe false, false, false, - None, false, ); diff --git a/src/query/sql/src/planner/metadata/metadata.rs b/src/query/sql/src/planner/metadata/metadata.rs index 11e16b1541f7f..f7889a42c8447 100644 --- a/src/query/sql/src/planner/metadata/metadata.rs +++ b/src/query/sql/src/planner/metadata/metadata.rs @@ -341,11 +341,9 @@ impl Metadata { source_of_view: bool, source_of_index: bool, source_of_stage: bool, - cte_suffix_name: Option, allow_virtual_column: bool, ) -> IndexType { let table_name = table_meta.name().to_string(); - let table_name = Self::remove_cte_suffix(table_name, cte_suffix_name); let table_index = self.tables.len(); // If exists table alias name, use it instead of origin name @@ -539,15 +537,6 @@ impl Metadata { self.base_column_scan_id.get(&column_index).cloned() } - fn remove_cte_suffix(mut table_name: String, cte_suffix_name: Option) -> String { - if let Some(suffix) = cte_suffix_name { - if table_name.ends_with(&suffix) { - table_name.truncate(table_name.len() - suffix.len() - 1); - } - } - table_name - } - pub fn replace_all_tables(&mut self, table: Arc) { for entry in self.tables.iter_mut() { entry.table = table.clone(); diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index af3450e008e78..3cedd1cb6cda5 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -13,41 +13,24 @@ // limitations under the License. use std::hash::Hash; -use std::hash::Hasher; - -use databend_common_expression::DataField; use crate::plans::Operator; use crate::plans::RelOp; -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct CTEConsumer { pub cte_name: String, - pub fields: Vec, -} - -impl CTEConsumer { - pub fn new(cte_name: String, fields: Vec) -> Self { - Self { cte_name, fields } - } - - // pub fn used_columns(&self) -> Result { - // let mut used_columns = ColumnSet::new(); - // for field in self.fields.iter() { - // used_columns.insert(field.name().parse()?); - // } - // Ok(used_columns) - // } } -impl Hash for CTEConsumer { - fn hash(&self, state: &mut H) { - self.cte_name.hash(state); - for field in self.fields.iter() { - field.name().hash(state); - } - } -} +// impl CTEConsumer { +// // pub fn used_columns(&self) -> Result { +// // let mut used_columns = ColumnSet::new(); +// // for field in self.fields.iter() { +// // used_columns.insert(field.name().parse()?); +// // } +// // Ok(used_columns) +// // } +// } impl Operator for CTEConsumer { fn rel_op(&self) -> RelOp { diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index b7793e8bd3546..3a4193c43926a 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -13,12 +13,11 @@ // limitations under the License. use std::hash::Hash; -use std::hash::Hasher; use crate::plans::Operator; use crate::plans::RelOp; -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct MaterializedCTE { pub cte_name: String, } @@ -29,12 +28,6 @@ impl MaterializedCTE { } } -impl Hash for MaterializedCTE { - fn hash(&self, state: &mut H) { - self.cte_name.hash(state); - } -} - impl Operator for MaterializedCTE { fn rel_op(&self) -> RelOp { RelOp::MaterializedCTE diff --git a/src/query/sql/src/planner/plans/operator.rs b/src/query/sql/src/planner/plans/operator.rs index d31852361cfd4..72d23ace98c73 100644 --- a/src/query/sql/src/planner/plans/operator.rs +++ b/src/query/sql/src/planner/plans/operator.rs @@ -970,3 +970,23 @@ impl TryFrom for MutationSource { } } } + +impl From for RelOperator { + fn from(v: MaterializedCTE) -> Self { + Self::MaterializedCTE(v) + } +} + +impl TryFrom for MaterializedCTE { + type Error = ErrorCode; + fn try_from(value: RelOperator) -> Result { + if let RelOperator::MaterializedCTE(value) = value { + Ok(value) + } else { + Err(ErrorCode::Internal(format!( + "Cannot downcast {:?} to MaterializedCTE", + value.rel_op() + ))) + } + } +} From e5d7472b467e8eced8bcc9b291b5929a330d5813 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 23 Jun 2025 22:48:06 +0800 Subject: [PATCH 07/37] fix --- .../src/pipelines/builders/builder_materialized_cte.rs | 8 ++------ src/query/service/src/sessions/query_ctx.rs | 2 -- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 02e42f3ee885f..01db9afb7cf61 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -25,12 +25,8 @@ impl PipelineBuilder { pub(crate) fn build_materialized_cte(&mut self, cte: &MaterializedCTE) -> Result<()> { // init builder for cte pipeline let sub_context = QueryContext::create_from(self.ctx.as_ref()); - let sub_builder = PipelineBuilder::create( - self.func_ctx.clone(), - self.settings.clone(), - sub_context, - self.main_pipeline.get_scopes(), - ); + let sub_builder = + PipelineBuilder::create(self.func_ctx.clone(), self.settings.clone(), sub_context); // build cte pipeline let mut build_res = sub_builder.finalize(&cte.left)?; diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index 10652f5e93ad9..1d5731a56f6d9 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -120,8 +120,6 @@ use databend_common_storages_stream::stream_table::StreamTable; use databend_common_users::GrantObjectVisibilityChecker; use databend_common_users::UserApiProvider; use databend_common_version::DATABEND_COMMIT_VERSION; -use databend_common_version::DATABEND_ENTERPRISE_LICENSE_EMBEDDED; -use databend_storages_common_session::drop_table_by_id; use databend_storages_common_session::SessionState; use databend_storages_common_session::TxnManagerRef; use databend_storages_common_table_meta::meta::Location; From 9a53ba2d80413f5322c9dbb3a9821b18c5b5b3b3 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Sun, 29 Jun 2025 17:10:02 +0800 Subject: [PATCH 08/37] remove unused field --- src/query/sql/src/planner/binder/bind_context.rs | 1 - src/query/sql/src/planner/binder/bind_query/bind.rs | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 3f74fe384b8a6..d1ac8e3430a99 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -197,7 +197,6 @@ pub struct CteInfo { pub query: Query, pub materialized: bool, pub recursive: bool, - pub cte_idx: IndexType, // If cte is materialized, save its columns pub columns: Vec, } diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 83d3b0a6e16b2..9ef31759beab9 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -135,7 +135,7 @@ impl Binder { return Ok(()); }; - for (idx, cte) in with.ctes.iter().enumerate() { + for cte in with.ctes.iter() { let table_name = self.normalize_identifier(&cte.alias.name).name; if bind_context.cte_context.cte_map.contains_key(&table_name) { return Err(ErrorCode::SemanticError(format!( @@ -152,7 +152,6 @@ impl Binder { columns_alias: column_name, query: *cte.query.clone(), recursive: with.recursive, - cte_idx: idx, columns: vec![], materialized: cte.materialized, }; From ff73950677075a6635e6e3de9d3a811a877f94d1 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 08:59:54 +0800 Subject: [PATCH 09/37] fix bind --- .../binder/bind_table_reference/bind_table.rs | 81 ++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 685cbf6f6eaf5..c8fd57af473a5 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::BTreeMap; use std::sync::Arc; +use dashmap::DashMap; use databend_common_ast::ast::Identifier; use databend_common_ast::ast::SampleConfig; use databend_common_ast::ast::Statement; @@ -34,10 +36,12 @@ use databend_storages_common_table_meta::table::get_change_type; use crate::binder::util::TableIdentifier; use crate::binder::Binder; +use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::plans::CTEConsumer; use crate::plans::RelOperator; use crate::BindContext; +use crate::binder::ExprContext; impl Binder { /// Bind a base table. @@ -76,10 +80,85 @@ impl Binder { let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { if cte_info.materialized { + // For materialized CTE, we need to add column bindings to the bind_context + // so that column references can be resolved correctly + let mut new_bind_context = bind_context.clone(); + + // We need to bind the CTE definition to get the columns + // This is similar to how non-materialized CTEs are handled + let mut cte_bind_context = BindContext { + parent: Some(Box::new(bind_context.clone())), + bound_internal_columns: BTreeMap::new(), + columns: vec![], + aggregate_info: Default::default(), + windows: Default::default(), + srf_info: Default::default(), + cte_context: bind_context.cte_context.clone(), + in_grouping: false, + view_info: None, + have_async_func: false, + have_udf_script: false, + have_udf_server: false, + inverted_index_map: Box::default(), + allow_virtual_column: false, + expr_context: ExprContext::default(), + planning_agg_index: false, + window_definitions: DashMap::new(), + }; + + cte_bind_context.cte_context.cte_name = Some(table_name.to_string()); + + // Bind the CTE definition to get the columns + let (_, mut res_bind_context) = + self.bind_query(&mut cte_bind_context, &cte_info.query)?; + + // Apply column aliases + let mut cols_alias = cte_info.columns_alias.clone(); + if let Some(alias) = alias { + for (idx, col_alias) in alias.columns.iter().enumerate() { + if idx < cte_info.columns_alias.len() { + cols_alias[idx] = col_alias.name.clone(); + } else { + cols_alias.push(col_alias.name.clone()); + } + } + } + + let alias_table_name = alias + .as_ref() + .map(|alias| normalize_identifier(&alias.name, &self.name_resolution_ctx).name) + .unwrap_or_else(|| table_name.to_string()); + + for column in res_bind_context.columns.iter_mut() { + column.database_name = None; + column.table_name = Some(alias_table_name.clone()); + } + + if cols_alias.len() > res_bind_context.columns.len() { + return Err(ErrorCode::SemanticError(format!( + "The CTE '{}' has {} columns, but {} aliases were provided. Ensure the number of aliases matches the number of columns in the CTE.", + table_name, + res_bind_context.columns.len(), + cols_alias.len() + )) + .set_span(*span)); + } + + for (index, column_name) in cols_alias.iter().enumerate() { + res_bind_context.columns[index].column_name = column_name.clone(); + } + + log::info!("[CTE] columns: {:?}", res_bind_context.columns); + + // Add the columns to the new bind context + for column in res_bind_context.columns { + new_bind_context.add_column_binding(column); + } + let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { cte_name: table_name, }))); - return Ok((s_expr, bind_context.clone())); + return Ok((s_expr, new_bind_context)); } else { if self .metadata From 519713429e9379142d8ce04e2b808727188d5d6e Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 09:50:56 +0800 Subject: [PATCH 10/37] fix schema --- .../physical_plans/physical_cte_consumer.rs | 2 +- .../binder/bind_table_reference/bind_table.rs | 18 +++++++++++++++-- .../sql/src/planner/plans/cte_consumer.rs | 20 +++++++++---------- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs index 819c5437ac5cb..cfb418c43fae2 100644 --- a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs +++ b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs @@ -46,7 +46,7 @@ impl PhysicalPlanBuilder { plan_id: 0, stat_info: Some(stat_info), cte_name: cte_consumer.cte_name.clone(), - cte_schema: Default::default(), + cte_schema: cte_consumer.cte_schema.clone(), }))) } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index c8fd57af473a5..66489f324b5fa 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -31,18 +31,19 @@ use databend_common_catalog::table_with_options::get_with_opt_consume; use databend_common_catalog::table_with_options::get_with_opt_max_batch_size; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::DataField; +use databend_common_expression::DataSchemaRefExt; use databend_common_storages_view::view_table::QUERY; use databend_storages_common_table_meta::table::get_change_type; use crate::binder::util::TableIdentifier; use crate::binder::Binder; +use crate::binder::ExprContext; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::plans::CTEConsumer; use crate::plans::RelOperator; use crate::BindContext; -use crate::binder::ExprContext; - impl Binder { /// Bind a base table. /// A base table is a table that is not a view or CTE. @@ -148,6 +149,18 @@ impl Binder { res_bind_context.columns[index].column_name = column_name.clone(); } + let fields = res_bind_context + .columns + .iter() + .map(|column_binding| { + DataField::new( + &column_binding.index.to_string(), + *column_binding.data_type.clone(), + ) + }) + .collect(); + let cte_schema = DataSchemaRefExt::create(fields); + log::info!("[CTE] columns: {:?}", res_bind_context.columns); // Add the columns to the new bind context @@ -157,6 +170,7 @@ impl Binder { let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { cte_name: table_name, + cte_schema: cte_schema, }))); return Ok((s_expr, new_bind_context)); } else { diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index 3cedd1cb6cda5..bb161ac105829 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -14,23 +14,23 @@ use std::hash::Hash; +use databend_common_expression::DataSchemaRef; + use crate::plans::Operator; use crate::plans::RelOp; +use std::hash::Hasher; -#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct CTEConsumer { pub cte_name: String, + pub cte_schema: DataSchemaRef, } -// impl CTEConsumer { -// // pub fn used_columns(&self) -> Result { -// // let mut used_columns = ColumnSet::new(); -// // for field in self.fields.iter() { -// // used_columns.insert(field.name().parse()?); -// // } -// // Ok(used_columns) -// // } -// } +impl Hash for CTEConsumer { + fn hash(&self, state: &mut H) { + self.cte_name.hash(state); + } +} impl Operator for CTEConsumer { fn rel_op(&self) -> RelOp { From ba5be420881941de370d41c878911e832e0aa05f Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 16:03:59 +0800 Subject: [PATCH 11/37] fix --- src/query/sql/src/executor/physical_plan_builder.rs | 2 +- .../executor/physical_plans/physical_materialized_cte.rs | 6 ++++-- src/query/sql/src/planner/binder/bind_query/bind.rs | 4 ++-- src/query/sql/src/planner/plans/materialized_cte.rs | 6 ++++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/query/sql/src/executor/physical_plan_builder.rs b/src/query/sql/src/executor/physical_plan_builder.rs index cc866a3f44b36..aa948994a22b5 100644 --- a/src/query/sql/src/executor/physical_plan_builder.rs +++ b/src/query/sql/src/executor/physical_plan_builder.rs @@ -130,7 +130,7 @@ impl PhysicalPlanBuilder { } RelOperator::CompactBlock(compact) => self.build_compact_block(compact).await, RelOperator::MaterializedCTE(materialized_cte) => { - self.build_materialized_cte(s_expr, materialized_cte, stat_info) + self.build_materialized_cte(s_expr, materialized_cte, stat_info, required) .await } RelOperator::CTEConsumer(cte_consumer) => { diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index 6b42cba900749..f34abf156aabd 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -19,6 +19,7 @@ use crate::executor::explain::PlanStatsInfo; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; +use crate::ColumnSet; /// This is a binary operator that executes its children in order (left to right), and returns the results of the right child #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -44,9 +45,10 @@ impl PhysicalPlanBuilder { s_expr: &SExpr, materialized_cte: &crate::plans::MaterializedCTE, stat_info: PlanStatsInfo, + required: ColumnSet, ) -> Result { - let left_side = Box::new(self.build(s_expr.child(0)?, Default::default()).await?); - let right_side = Box::new(self.build(s_expr.child(1)?, Default::default()).await?); + let left_side = Box::new(self.build(s_expr.child(0)?, materialized_cte.required.clone()).await?); + let right_side = Box::new(self.build(s_expr.child(1)?, required).await?); Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { plan_id: 0, stat_info: Some(stat_info), diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 9ef31759beab9..988e98a76fc02 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -252,11 +252,11 @@ impl Binder { }; // Bind the CTE definition - let (cte_definition_expr, _) = + let (cte_definition_expr, bind_context) = self.bind_query(&mut cte_bind_context, &cte.query)?; // Create the MaterializedCTE operator - let materialized_cte = MaterializedCTE::new(cte_name); + let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); current_expr = SExpr::create_binary( Arc::new(materialized_cte.into()), Arc::new(cte_definition_expr), diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index 3a4193c43926a..c37b286ef9436 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -16,15 +16,17 @@ use std::hash::Hash; use crate::plans::Operator; use crate::plans::RelOp; +use crate::ColumnSet; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct MaterializedCTE { pub cte_name: String, + pub required: ColumnSet, } impl MaterializedCTE { - pub fn new(cte_name: String) -> Self { - Self { cte_name } + pub fn new(cte_name: String, required: ColumnSet) -> Self { + Self { cte_name, required } } } From 7690dbf8657c0ff7e1031d6e40e4713a18a0c764 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 16:14:20 +0800 Subject: [PATCH 12/37] make lint --- .../src/executor/physical_plans/physical_materialized_cte.rs | 5 ++++- .../src/planner/binder/bind_table_reference/bind_table.rs | 2 +- src/query/sql/src/planner/plans/cte_consumer.rs | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index f34abf156aabd..401278e26280d 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -47,7 +47,10 @@ impl PhysicalPlanBuilder { stat_info: PlanStatsInfo, required: ColumnSet, ) -> Result { - let left_side = Box::new(self.build(s_expr.child(0)?, materialized_cte.required.clone()).await?); + let left_side = Box::new( + self.build(s_expr.child(0)?, materialized_cte.required.clone()) + .await?, + ); let right_side = Box::new(self.build(s_expr.child(1)?, required).await?); Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { plan_id: 0, diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 66489f324b5fa..8b7ae6d7cb160 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -170,7 +170,7 @@ impl Binder { let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { cte_name: table_name, - cte_schema: cte_schema, + cte_schema, }))); return Ok((s_expr, new_bind_context)); } else { diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index bb161ac105829..d0aa0c0ec6020 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -13,12 +13,12 @@ // limitations under the License. use std::hash::Hash; +use std::hash::Hasher; use databend_common_expression::DataSchemaRef; use crate::plans::Operator; use crate::plans::RelOp; -use std::hash::Hasher; #[derive(Clone, Debug, PartialEq, Eq)] pub struct CTEConsumer { From f8f4d7abb13defce72a791cd3efc20320e4d2670 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 16:55:37 +0800 Subject: [PATCH 13/37] fix --- src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs index f587d69c5ca1a..a63586634106e 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs @@ -487,10 +487,13 @@ impl SExpr { | crate::plans::RelOp::ConstantTableScan | crate::plans::RelOp::ExpressionScan | crate::plans::RelOp::CacheScan + | crate::plans::RelOp::CTEConsumer | crate::plans::RelOp::RecursiveCteScan => Ok(None), crate::plans::RelOp::Join => self.probe_side_child().get_data_distribution(), + crate::plans::RelOp::MaterializedCTE => self.child(1)?.get_data_distribution(), + crate::plans::RelOp::Exchange => { Ok(Some(self.plan.as_ref().clone().try_into().unwrap())) } From 4c75ded3978fd36e012ae2dd3f07115eceb8130f Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 20:36:14 +0800 Subject: [PATCH 14/37] fix join --- .../builders/builder_cte_consumer.rs | 20 ++++++++----------- .../src/pipelines/builders/builder_join.rs | 2 ++ .../builders/builder_materialized_cte.rs | 8 ++++++-- .../service/src/pipelines/pipeline_builder.rs | 5 +++-- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs index adfe50a2b0fad..1fa78f91fbc7d 100644 --- a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs +++ b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs @@ -30,19 +30,15 @@ impl PipelineBuilder { })? .clone(); - let current_consumer_id = - *self - .next_cte_consumer_id - .get(&cte.cte_name) - .ok_or_else(|| { - ErrorCode::Internal(format!( - "CTE consumer id not found for name: {}", - cte.cte_name - )) - })?; + let mut next_cte_consumer_id = self.next_cte_consumer_id.lock(); + let current_consumer_id = *next_cte_consumer_id.get(&cte.cte_name).ok_or_else(|| { + ErrorCode::Internal(format!( + "CTE consumer id not found for name: {}", + cte.cte_name + )) + })?; - self.next_cte_consumer_id - .insert(cte.cte_name.clone(), current_consumer_id + 1); + next_cte_consumer_id.insert(cte.cte_name.clone(), current_consumer_id + 1); self.main_pipeline.add_source( |output_port| { diff --git a/src/query/service/src/pipelines/builders/builder_join.rs b/src/query/service/src/pipelines/builders/builder_join.rs index 7937b11b3344c..09b1b9a7bb3d7 100644 --- a/src/query/service/src/pipelines/builders/builder_join.rs +++ b/src/query/service/src/pipelines/builders/builder_join.rs @@ -41,6 +41,8 @@ impl PipelineBuilder { let mut sub_builder = PipelineBuilder::create(self.func_ctx.clone(), self.settings.clone(), sub_context); sub_builder.hash_join_states = self.hash_join_states.clone(); + sub_builder.cte_receivers = self.cte_receivers.clone(); + sub_builder.next_cte_consumer_id = self.next_cte_consumer_id.clone(); sub_builder } diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 01db9afb7cf61..4336c2691920e 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -25,15 +25,19 @@ impl PipelineBuilder { pub(crate) fn build_materialized_cte(&mut self, cte: &MaterializedCTE) -> Result<()> { // init builder for cte pipeline let sub_context = QueryContext::create_from(self.ctx.as_ref()); - let sub_builder = + let mut sub_builder = PipelineBuilder::create(self.func_ctx.clone(), self.settings.clone(), sub_context); + sub_builder.cte_receivers = self.cte_receivers.clone(); + sub_builder.next_cte_consumer_id = self.next_cte_consumer_id.clone(); // build cte pipeline let mut build_res = sub_builder.finalize(&cte.left)?; build_res.main_pipeline.try_resize(1)?; let (tx, rx) = tokio::sync::watch::channel(Arc::new(MaterializedCteData::default())); self.cte_receivers.insert(cte.cte_name.clone(), rx); - self.next_cte_consumer_id.insert(cte.cte_name.clone(), 0); + self.next_cte_consumer_id + .lock() + .insert(cte.cte_name.clone(), 0); build_res .main_pipeline .add_sink(|input| MaterializedCteSink::create(input, tx.clone()))?; diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index ac7262cb90fbc..c3b909d72075c 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -27,6 +27,7 @@ use databend_common_pipeline_core::ExecutionInfo; use databend_common_pipeline_core::Pipeline; use databend_common_settings::Settings; use databend_common_sql::executor::PhysicalPlan; +use parking_lot::Mutex; use tokio::sync::watch::Receiver; use super::PipelineBuilderData; @@ -60,7 +61,7 @@ pub struct PipelineBuilder { pub contain_sink_processor: bool, pub cte_receivers: HashMap>>, - pub next_cte_consumer_id: HashMap, + pub next_cte_consumer_id: Arc>>, } impl PipelineBuilder { @@ -83,7 +84,7 @@ impl PipelineBuilder { contain_sink_processor: false, is_exchange_stack: vec![], cte_receivers: HashMap::new(), - next_cte_consumer_id: HashMap::new(), + next_cte_consumer_id: Arc::new(Mutex::new(HashMap::new())), } } From cc89312ada1fdf0811f7f36c612a30abf9bdfd63 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 23:54:20 +0800 Subject: [PATCH 15/37] fix --- src/query/sql/src/planner/binder/bind_query/bind.rs | 5 ++++- .../planner/binder/bind_table_reference/bind_table.rs | 9 +++++++++ src/query/sql/src/planner/binder/table.rs | 9 --------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 988e98a76fc02..1d7750bfdfd6c 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -245,9 +245,12 @@ impl Binder { if cte.materialized { let cte_name = self.normalize_identifier(&cte.alias.name).name; + let mut cte_context = cte_context.clone(); + cte_context.cte_name = Some(cte_name.clone()); + // Create a new bind context for the CTE definition let mut cte_bind_context = BindContext { - cte_context: cte_context.clone(), + cte_context, ..Default::default() }; diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 8b7ae6d7cb160..4a0610ac5c1f0 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -68,6 +68,15 @@ impl Binder { table_identifier.table_name_alias(), ); + if let Some(cte_name) = &bind_context.cte_context.cte_name { + if cte_name == &table_name { + return Err(ErrorCode::SemanticError(format!( + "The cte {table_name} is not recursive, but it references itself.", + )) + .set_span(*span)); + } + } + let (consume, max_batch_size, with_opts_str) = if let Some(with_options) = with_options { check_with_opt_valid(with_options)?; let consume = get_with_opt_consume(with_options)?; diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index 5420257a63797..fcd5ac7b6e82c 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -163,15 +163,6 @@ impl Binder { alias: &Option, cte_info: &CteInfo, ) -> Result<(SExpr, BindContext)> { - if let Some(cte_name) = &bind_context.cte_context.cte_name { - if cte_name == table_name { - return Err(ErrorCode::SemanticError(format!( - "The cte {table_name} is not recursive, but it references itself.", - )) - .set_span(span)); - } - } - let mut new_bind_context = BindContext { parent: Some(Box::new(bind_context.clone())), bound_internal_columns: BTreeMap::new(), From 291204aa7c4bcf6749af4e7acc5a4988ca293d93 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Tue, 1 Jul 2025 00:12:56 +0800 Subject: [PATCH 16/37] refine explain --- src/query/sql/src/executor/format.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index 8a5f11a61244a..0fa30d2531459 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -533,7 +533,7 @@ fn to_format_tree( children.push(to_format_tree(&plan.left, metadata, profs, context)?); children.push(to_format_tree(&plan.right, metadata, profs, context)?); Ok(FormatTreeNode::with_children( - "MaterializedCTE".to_string(), + format!("MaterializedCTE: {}", plan.cte_name), children, )) } From 4835fb85208993a4d201163efcd9cdfda14da63f Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Tue, 1 Jul 2025 19:30:31 +0800 Subject: [PATCH 17/37] fix --- .../sql/src/planner/binder/bind_context.rs | 5 ++- .../sql/src/planner/binder/bind_query/bind.rs | 11 ++++- .../binder/bind_table_reference/bind_table.rs | 9 +++- .../src/planner/optimizer/ir/expr/s_expr.rs | 17 ++++++++ .../sql/src/planner/plans/cte_consumer.rs | 43 ++++++------------- .../sql/src/planner/plans/materialized_cte.rs | 26 +++++++++++ 6 files changed, 76 insertions(+), 35 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index d1ac8e3430a99..b1493b1a78cc4 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -16,6 +16,7 @@ use std::collections::btree_map; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::hash::Hash; +use std::sync::Arc; use dashmap::DashMap; use databend_common_ast::ast::Identifier; @@ -47,6 +48,8 @@ use crate::ColumnSet; use crate::IndexType; use crate::MetadataRef; use crate::NameResolutionContext; +use std::sync::Mutex; +use crate::optimizer::ir::StatInfo; /// Context of current expression, this is used to check if /// the expression is valid in current context. @@ -160,7 +163,6 @@ pub struct CteContext { /// If the `BindContext` is created from a CTE, record the cte name pub cte_name: Option, pub cte_map: Box>, - pub is_binding_materialized_cte: bool, } impl CteContext { @@ -199,6 +201,7 @@ pub struct CteInfo { pub recursive: bool, // If cte is materialized, save its columns pub columns: Vec, + pub stat_info: Arc>>>, } impl BindContext { diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 1d7750bfdfd6c..e752d19faf0fe 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; use std::sync::Arc; +use std::sync::Mutex; use databend_common_ast::ast::Expr; use databend_common_ast::ast::Query; @@ -27,6 +28,7 @@ use derive_visitor::Visitor; use crate::binder::CteInfo; use crate::normalize_identifier; +use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::planner::binder::scalar::ScalarBinder; use crate::planner::binder::BindContext; @@ -154,6 +156,7 @@ impl Binder { recursive: with.recursive, columns: vec![], materialized: cte.materialized, + stat_info: Arc::new(Mutex::new(None)), }; bind_context .cte_context @@ -236,10 +239,9 @@ impl Binder { &mut self, with: &With, main_query_expr: SExpr, - mut cte_context: CteContext, + cte_context: CteContext, ) -> Result { let mut current_expr = main_query_expr; - cte_context.is_binding_materialized_cte = true; for cte in with.ctes.iter().rev() { if cte.materialized { @@ -247,6 +249,7 @@ impl Binder { let mut cte_context = cte_context.clone(); cte_context.cte_name = Some(cte_name.clone()); + let shared_stat_info = cte_context.cte_map.get(&cte_name).unwrap().stat_info.clone(); // Create a new bind context for the CTE definition let mut cte_bind_context = BindContext { @@ -265,6 +268,10 @@ impl Binder { Arc::new(cte_definition_expr), Arc::new(current_expr), ); + + // Derive cardinality statistics and share with consumer + let stats = RelExpr::with_s_expr(¤t_expr).derive_cardinality()?; + *shared_stat_info.lock().unwrap() = Some(stats); } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 4a0610ac5c1f0..faf51e525658d 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -177,10 +177,15 @@ impl Binder { new_bind_context.add_column_binding(column); } - let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { + let s_expr = SExpr::create_with_shared_stat_info(Arc::new(RelOperator::CTEConsumer(CTEConsumer { cte_name: table_name, cte_schema, - }))); + })), + vec![], + None, + None, + cte_info.stat_info.clone(), + ); return Ok((s_expr, new_bind_context)); } else { if self diff --git a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs index a63586634106e..3f89fddc445d0 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs @@ -83,6 +83,23 @@ impl SExpr { } } + pub fn create_with_shared_stat_info( + plan: Arc, + children: impl Into>>, + original_group: Option, + rel_prop: Option>, + stat_info: Arc>>>, + ) -> Self { + SExpr { + plan, + children: children.into(), + original_group, + rel_prop: Arc::new(Mutex::new(rel_prop)), + stat_info, + applied_rules: AppliedRules::default(), + } + } + pub fn create_unary(plan: Arc, child: impl Into>) -> Self { Self::create(plan, [child.into()], None, None, None) } diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index d0aa0c0ec6020..b9226478b5695 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -17,8 +17,13 @@ use std::hash::Hasher; use databend_common_expression::DataSchemaRef; +use crate::optimizer::ir::RelExpr; use crate::plans::Operator; use crate::plans::RelOp; +use std::sync::Arc; + +use databend_common_exception::Result; +use crate::optimizer::ir::StatInfo; #[derive(Clone, Debug, PartialEq, Eq)] pub struct CTEConsumer { @@ -37,35 +42,13 @@ impl Operator for CTEConsumer { RelOp::CTEConsumer } - // fn arity(&self) -> usize { - // 0 - // } - - // fn derive_relational_prop(&self, _rel_expr: &RelExpr) -> Result> { - // Ok(Arc::new(RelationalProperty { - // output_columns: self.used_columns()?, - // outer_columns: ColumnSet::new(), - // used_columns: self.used_columns()?, - // orderings: vec![], - // partition_orderings: None, - // })) - // } - - // fn derive_physical_prop(&self, _rel_expr: &RelExpr) -> Result { - // Ok(PhysicalProperty { - // distribution: Distribution::Serial, - // }) - // } + /// Get arity of this operator + fn arity(&self) -> usize { + 0 + } - // fn compute_required_prop_child( - // &self, - // _ctx: Arc, - // _rel_expr: &RelExpr, - // _child_index: usize, - // _required: &RequiredProperty, - // ) -> Result { - // Err(ErrorCode::Internal( - // "Cannot compute required property for CTEConsumer".to_string(), - // )) - // } + /// Derive statistics information + fn derive_stats(&self, rel_expr: &RelExpr) -> Result> { + rel_expr.derive_cardinality() + } } diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index c37b286ef9436..cbdefc6ccf2d5 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -14,10 +14,16 @@ use std::hash::Hash; +use crate::optimizer::ir::RelExpr; use crate::plans::Operator; use crate::plans::RelOp; use crate::ColumnSet; +use std::sync::Arc; +use databend_common_exception::Result; +use crate::optimizer::ir::PhysicalProperty; +use crate::optimizer::ir::RelationalProperty; +use crate::optimizer::ir::StatInfo; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct MaterializedCTE { pub cte_name: String, @@ -34,4 +40,24 @@ impl Operator for MaterializedCTE { fn rel_op(&self) -> RelOp { RelOp::MaterializedCTE } + + /// Get arity of this operator + fn arity(&self) -> usize { + 2 + } + + /// Derive relational property + fn derive_relational_prop(&self, rel_expr: &RelExpr) -> Result> { + rel_expr.derive_relational_prop_child(1) + } + + /// Derive physical property + fn derive_physical_prop(&self, rel_expr: &RelExpr) -> Result { + rel_expr.derive_physical_prop_child(1) + } + + /// Derive statistics information + fn derive_stats(&self, rel_expr: &RelExpr) -> Result> { + rel_expr.derive_cardinality_child(1) + } } From 046bd041150901f72781c50ab43cced0e1b93837 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 2 Jul 2025 16:12:00 +0800 Subject: [PATCH 18/37] fix --- .../sql/src/planner/binder/bind_context.rs | 14 +++- .../planner/binder/bind_mutation/delete.rs | 2 +- .../planner/binder/bind_mutation/update.rs | 2 +- .../sql/src/planner/binder/bind_query/bind.rs | 84 ++++++++++--------- .../binder/bind_table_reference/bind_table.rs | 70 ++++------------ .../src/planner/binder/copy_into_location.rs | 2 +- .../sql/src/planner/binder/copy_into_table.rs | 2 +- src/query/sql/src/planner/binder/insert.rs | 2 +- .../src/planner/optimizer/ir/expr/s_expr.rs | 17 ---- .../sql/src/planner/plans/cte_consumer.rs | 7 +- .../sql/src/planner/plans/materialized_cte.rs | 10 +-- 11 files changed, 85 insertions(+), 127 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index b1493b1a78cc4..688afe9424fd0 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -43,13 +43,13 @@ use crate::binder::project_set::SetReturningInfo; use crate::binder::window::WindowInfo; use crate::binder::ColumnBindingBuilder; use crate::normalize_identifier; +use crate::optimizer::ir::SExpr; +use crate::optimizer::ir::StatInfo; use crate::plans::ScalarExpr; use crate::ColumnSet; use crate::IndexType; use crate::MetadataRef; use crate::NameResolutionContext; -use std::sync::Mutex; -use crate::optimizer::ir::StatInfo; /// Context of current expression, this is used to check if /// the expression is valid in current context. @@ -197,11 +197,17 @@ impl CteContext { pub struct CteInfo { pub columns_alias: Vec, pub query: Query, + pub bind_result: CteBindResult, pub materialized: bool, pub recursive: bool, - // If cte is materialized, save its columns pub columns: Vec, - pub stat_info: Arc>>>, +} + +#[derive(Clone, Debug)] +pub struct CteBindResult { + pub s_expr: SExpr, + pub bind_context: BindContext, + pub stat_info: Arc, } impl BindContext { diff --git a/src/query/sql/src/planner/binder/bind_mutation/delete.rs b/src/query/sql/src/planner/binder/bind_mutation/delete.rs index 0e8fcb8d7d671..b1b0f07e74529 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/delete.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/delete.rs @@ -41,7 +41,7 @@ impl Binder { .. } = stamt; - self.init_cte(bind_context, with)?; + self.bind_cte_def(bind_context, with)?; let target_table_identifier = if let TableReference::Table { catalog, diff --git a/src/query/sql/src/planner/binder/bind_mutation/update.rs b/src/query/sql/src/planner/binder/bind_mutation/update.rs index 56de12c2c2921..c267af8dc25a5 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/update.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/update.rs @@ -63,7 +63,7 @@ impl Binder { .. } = stmt; - self.init_cte(bind_context, with)?; + self.bind_cte_def(bind_context, with)?; let target_table_identifier = TableIdentifier::new(self, catalog, database, table, table_alias); diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index e752d19faf0fe..9e1bb0c69c86f 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -14,7 +14,6 @@ use std::collections::HashMap; use std::sync::Arc; -use std::sync::Mutex; use databend_common_ast::ast::Expr; use databend_common_ast::ast::Query; @@ -26,6 +25,7 @@ use databend_common_exception::Result; use derive_visitor::Drive; use derive_visitor::Visitor; +use crate::binder::CteBindResult; use crate::binder::CteInfo; use crate::normalize_identifier; use crate::optimizer::ir::RelExpr; @@ -73,8 +73,9 @@ impl Binder { } } } - // Initialize cte map. - self.init_cte(bind_context, &with)?; + + // Bind cte definition. + self.bind_cte_def(bind_context, &with)?; // Extract limit and offset from query. let (limit, offset) = self.extract_limit_and_offset(query)?; @@ -125,43 +126,53 @@ impl Binder { Ok(()) } - // Initialize cte map. - pub(crate) fn init_cte( + pub(crate) fn bind_cte_def( &mut self, bind_context: &mut BindContext, with: &Option, ) -> Result<()> { - let with = if let Some(with) = with { - with - } else { + let Some(with) = with else { return Ok(()); }; - for cte in with.ctes.iter() { - let table_name = self.normalize_identifier(&cte.alias.name).name; - if bind_context.cte_context.cte_map.contains_key(&table_name) { + let cte_name = self.normalize_identifier(&cte.alias.name).name; + if bind_context.cte_context.cte_map.contains_key(&cte_name) { return Err(ErrorCode::SemanticError(format!( - "Duplicate common table expression: {table_name}" + "Duplicate common table expression: {cte_name}" ))); } + let column_name = cte .alias .columns .iter() .map(|ident| self.normalize_identifier(ident).name) .collect(); + + let mut cte_bind_context = BindContext { + cte_context: CteContext { + cte_name: Some(cte_name.clone()), + cte_map: bind_context.cte_context.cte_map.clone(), + }, + ..Default::default() + }; + let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, &cte.query)?; + let stat_info = RelExpr::with_s_expr(&s_expr).derive_cardinality()?; + let bind_result = CteBindResult { + s_expr, + bind_context: cte_bind_context, + stat_info, + }; + let cte_info = CteInfo { columns_alias: column_name, query: *cte.query.clone(), recursive: with.recursive, - columns: vec![], materialized: cte.materialized, - stat_info: Arc::new(Mutex::new(None)), + columns: vec![], + bind_result, }; - bind_context - .cte_context - .cte_map - .insert(table_name, cte_info); + bind_context.cte_context.cte_map.insert(cte_name, cte_info); } Ok(()) @@ -246,32 +257,25 @@ impl Binder { for cte in with.ctes.iter().rev() { if cte.materialized { let cte_name = self.normalize_identifier(&cte.alias.name).name; + let CteBindResult { + s_expr, + bind_context, + stat_info, + } = cte_context + .cte_map + .get(&cte_name) + .unwrap() + .bind_result + .clone(); - let mut cte_context = cte_context.clone(); - cte_context.cte_name = Some(cte_name.clone()); - let shared_stat_info = cte_context.cte_map.get(&cte_name).unwrap().stat_info.clone(); - - // Create a new bind context for the CTE definition - let mut cte_bind_context = BindContext { - cte_context, - ..Default::default() - }; - - // Bind the CTE definition - let (cte_definition_expr, bind_context) = - self.bind_query(&mut cte_bind_context, &cte.query)?; - - // Create the MaterializedCTE operator let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); - current_expr = SExpr::create_binary( + current_expr = SExpr::create( Arc::new(materialized_cte.into()), - Arc::new(cte_definition_expr), - Arc::new(current_expr), + vec![Arc::new(s_expr), Arc::new(current_expr)], + None, + None, + Some(stat_info), ); - - // Derive cardinality statistics and share with consumer - let stats = RelExpr::with_s_expr(¤t_expr).derive_cardinality()?; - *shared_stat_info.lock().unwrap() = Some(stats); } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index faf51e525658d..b1da2f35ca22a 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -12,10 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::BTreeMap; use std::sync::Arc; -use dashmap::DashMap; use databend_common_ast::ast::Identifier; use databend_common_ast::ast::SampleConfig; use databend_common_ast::ast::Statement; @@ -38,7 +36,6 @@ use databend_storages_common_table_meta::table::get_change_type; use crate::binder::util::TableIdentifier; use crate::binder::Binder; -use crate::binder::ExprContext; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::plans::CTEConsumer; @@ -90,38 +87,7 @@ impl Binder { let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { if cte_info.materialized { - // For materialized CTE, we need to add column bindings to the bind_context - // so that column references can be resolved correctly - let mut new_bind_context = bind_context.clone(); - - // We need to bind the CTE definition to get the columns - // This is similar to how non-materialized CTEs are handled - let mut cte_bind_context = BindContext { - parent: Some(Box::new(bind_context.clone())), - bound_internal_columns: BTreeMap::new(), - columns: vec![], - aggregate_info: Default::default(), - windows: Default::default(), - srf_info: Default::default(), - cte_context: bind_context.cte_context.clone(), - in_grouping: false, - view_info: None, - have_async_func: false, - have_udf_script: false, - have_udf_server: false, - inverted_index_map: Box::default(), - allow_virtual_column: false, - expr_context: ExprContext::default(), - planning_agg_index: false, - window_definitions: DashMap::new(), - }; - - cte_bind_context.cte_context.cte_name = Some(table_name.to_string()); - - // Bind the CTE definition to get the columns - let (_, mut res_bind_context) = - self.bind_query(&mut cte_bind_context, &cte_info.query)?; - + let mut cte_bind_context = cte_info.bind_result.bind_context.clone(); // Apply column aliases let mut cols_alias = cte_info.columns_alias.clone(); if let Some(alias) = alias { @@ -139,26 +105,26 @@ impl Binder { .map(|alias| normalize_identifier(&alias.name, &self.name_resolution_ctx).name) .unwrap_or_else(|| table_name.to_string()); - for column in res_bind_context.columns.iter_mut() { + for column in cte_bind_context.columns.iter_mut() { column.database_name = None; column.table_name = Some(alias_table_name.clone()); } - if cols_alias.len() > res_bind_context.columns.len() { + if cols_alias.len() > cte_bind_context.columns.len() { return Err(ErrorCode::SemanticError(format!( "The CTE '{}' has {} columns, but {} aliases were provided. Ensure the number of aliases matches the number of columns in the CTE.", table_name, - res_bind_context.columns.len(), + cte_bind_context.columns.len(), cols_alias.len() )) .set_span(*span)); } for (index, column_name) in cols_alias.iter().enumerate() { - res_bind_context.columns[index].column_name = column_name.clone(); + cte_bind_context.columns[index].column_name = column_name.clone(); } - let fields = res_bind_context + let fields = cte_bind_context .columns .iter() .map(|column_binding| { @@ -170,22 +136,22 @@ impl Binder { .collect(); let cte_schema = DataSchemaRefExt::create(fields); - log::info!("[CTE] columns: {:?}", res_bind_context.columns); - // Add the columns to the new bind context - for column in res_bind_context.columns { + let mut new_bind_context = bind_context.clone(); + for column in cte_bind_context.columns { new_bind_context.add_column_binding(column); } - let s_expr = SExpr::create_with_shared_stat_info(Arc::new(RelOperator::CTEConsumer(CTEConsumer { - cte_name: table_name, - cte_schema, - })), - vec![], - None, - None, - cte_info.stat_info.clone(), - ); + let s_expr = SExpr::create( + Arc::new(RelOperator::CTEConsumer(CTEConsumer { + cte_name: table_name, + cte_schema, + })), + vec![], + None, + None, + Some(cte_info.bind_result.stat_info.clone()), + ); return Ok((s_expr, new_bind_context)); } else { if self diff --git a/src/query/sql/src/planner/binder/copy_into_location.rs b/src/query/sql/src/planner/binder/copy_into_location.rs index a6dd9e8d83c7a..01171a5102ea0 100644 --- a/src/query/sql/src/planner/binder/copy_into_location.rs +++ b/src/query/sql/src/planner/binder/copy_into_location.rs @@ -105,7 +105,7 @@ impl Binder { } } CopyIntoLocationSource::Query(query) => { - self.init_cte(bind_context, &stmt.with)?; + self.bind_cte_def(bind_context, &stmt.with)?; self.bind_statement(bind_context, &Statement::Query(query.clone())) .await } diff --git a/src/query/sql/src/planner/binder/copy_into_table.rs b/src/query/sql/src/planner/binder/copy_into_table.rs index 4c2652759705b..d10dea193440c 100644 --- a/src/query/sql/src/planner/binder/copy_into_table.rs +++ b/src/query/sql/src/planner/binder/copy_into_table.rs @@ -99,7 +99,7 @@ impl Binder { .await } CopyIntoTableSource::Query(query) => { - self.init_cte(bind_context, &stmt.with)?; + self.bind_cte_def(bind_context, &stmt.with)?; let mut max_column_position = MaxColumnPosition::new(); query.drive(&mut max_column_position); diff --git a/src/query/sql/src/planner/binder/insert.rs b/src/query/sql/src/planner/binder/insert.rs index 9cbfd4dc9b7df..aa374835cbb82 100644 --- a/src/query/sql/src/planner/binder/insert.rs +++ b/src/query/sql/src/planner/binder/insert.rs @@ -111,7 +111,7 @@ impl Binder { .. } = stmt; - self.init_cte(bind_context, with)?; + self.bind_cte_def(bind_context, with)?; let table_identifier = TableIdentifier::new(self, catalog, database, table, &None); let (catalog_name, database_name, table_name) = ( diff --git a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs index 3f89fddc445d0..a63586634106e 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs @@ -83,23 +83,6 @@ impl SExpr { } } - pub fn create_with_shared_stat_info( - plan: Arc, - children: impl Into>>, - original_group: Option, - rel_prop: Option>, - stat_info: Arc>>>, - ) -> Self { - SExpr { - plan, - children: children.into(), - original_group, - rel_prop: Arc::new(Mutex::new(rel_prop)), - stat_info, - applied_rules: AppliedRules::default(), - } - } - pub fn create_unary(plan: Arc, child: impl Into>) -> Self { Self::create(plan, [child.into()], None, None, None) } diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index b9226478b5695..41ed7ef60eb65 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -14,16 +14,15 @@ use std::hash::Hash; use std::hash::Hasher; +use std::sync::Arc; +use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::optimizer::ir::RelExpr; +use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; -use std::sync::Arc; - -use databend_common_exception::Result; -use crate::optimizer::ir::StatInfo; #[derive(Clone, Debug, PartialEq, Eq)] pub struct CTEConsumer { diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index cbdefc6ccf2d5..dba74c7f23d49 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -13,17 +13,17 @@ // limitations under the License. use std::hash::Hash; - -use crate::optimizer::ir::RelExpr; -use crate::plans::Operator; -use crate::plans::RelOp; -use crate::ColumnSet; use std::sync::Arc; use databend_common_exception::Result; + use crate::optimizer::ir::PhysicalProperty; +use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::StatInfo; +use crate::plans::Operator; +use crate::plans::RelOp; +use crate::ColumnSet; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct MaterializedCTE { pub cte_name: String, From 4979e0b62a708611ff282e0aac93a2628432f80a Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 2 Jul 2025 17:32:42 +0800 Subject: [PATCH 19/37] fix --- src/query/sql/src/planner/binder/bind_context.rs | 3 --- .../sql/src/planner/binder/bind_query/bind.rs | 12 +++--------- .../binder/bind_table_reference/bind_table.rs | 15 +++++---------- src/query/sql/src/planner/plans/cte_consumer.rs | 6 ++++-- 4 files changed, 12 insertions(+), 24 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 688afe9424fd0..544c64f2f7339 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -16,7 +16,6 @@ use std::collections::btree_map; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::hash::Hash; -use std::sync::Arc; use dashmap::DashMap; use databend_common_ast::ast::Identifier; @@ -44,7 +43,6 @@ use crate::binder::window::WindowInfo; use crate::binder::ColumnBindingBuilder; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; -use crate::optimizer::ir::StatInfo; use crate::plans::ScalarExpr; use crate::ColumnSet; use crate::IndexType; @@ -207,7 +205,6 @@ pub struct CteInfo { pub struct CteBindResult { pub s_expr: SExpr, pub bind_context: BindContext, - pub stat_info: Arc, } impl BindContext { diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 9e1bb0c69c86f..71ec123b54fe9 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -28,7 +28,6 @@ use derive_visitor::Visitor; use crate::binder::CteBindResult; use crate::binder::CteInfo; use crate::normalize_identifier; -use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::planner::binder::scalar::ScalarBinder; use crate::planner::binder::BindContext; @@ -157,11 +156,9 @@ impl Binder { ..Default::default() }; let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, &cte.query)?; - let stat_info = RelExpr::with_s_expr(&s_expr).derive_cardinality()?; let bind_result = CteBindResult { s_expr, bind_context: cte_bind_context, - stat_info, }; let cte_info = CteInfo { @@ -260,7 +257,6 @@ impl Binder { let CteBindResult { s_expr, bind_context, - stat_info, } = cte_context .cte_map .get(&cte_name) @@ -269,12 +265,10 @@ impl Binder { .clone(); let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); - current_expr = SExpr::create( + current_expr = SExpr::create_binary( Arc::new(materialized_cte.into()), - vec![Arc::new(s_expr), Arc::new(current_expr)], - None, - None, - Some(stat_info), + Arc::new(s_expr), + Arc::new(current_expr), ); } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index b1da2f35ca22a..113c33a54333e 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -142,16 +142,11 @@ impl Binder { new_bind_context.add_column_binding(column); } - let s_expr = SExpr::create( - Arc::new(RelOperator::CTEConsumer(CTEConsumer { - cte_name: table_name, - cte_schema, - })), - vec![], - None, - None, - Some(cte_info.bind_result.stat_info.clone()), - ); + let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { + cte_name: table_name, + cte_schema, + def: cte_info.bind_result.s_expr.clone(), + }))); return Ok((s_expr, new_bind_context)); } else { if self diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index 41ed7ef60eb65..e439b5b558636 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -20,6 +20,7 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::optimizer::ir::RelExpr; +use crate::optimizer::ir::SExpr; use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; @@ -28,6 +29,7 @@ use crate::plans::RelOp; pub struct CTEConsumer { pub cte_name: String, pub cte_schema: DataSchemaRef, + pub def: SExpr, } impl Hash for CTEConsumer { @@ -47,7 +49,7 @@ impl Operator for CTEConsumer { } /// Derive statistics information - fn derive_stats(&self, rel_expr: &RelExpr) -> Result> { - rel_expr.derive_cardinality() + fn derive_stats(&self, _rel_expr: &RelExpr) -> Result> { + RelExpr::with_s_expr(&self.def).derive_cardinality() } } From af0eeb7f3bb22887508a9e90e97b65541a87d80d Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 2 Jul 2025 17:51:11 +0800 Subject: [PATCH 20/37] fix --- .../sql/src/planner/binder/bind_context.rs | 2 +- .../sql/src/planner/binder/bind_query/bind.rs | 30 +++++++++++-------- .../binder/bind_table_reference/bind_table.rs | 4 +-- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 544c64f2f7339..790ab40698d8f 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -195,7 +195,7 @@ impl CteContext { pub struct CteInfo { pub columns_alias: Vec, pub query: Query, - pub bind_result: CteBindResult, + pub bind_result: Option, pub materialized: bool, pub recursive: bool, pub columns: Vec, diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 71ec123b54fe9..e2c6473bc0c0f 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -148,17 +148,22 @@ impl Binder { .map(|ident| self.normalize_identifier(ident).name) .collect(); - let mut cte_bind_context = BindContext { - cte_context: CteContext { - cte_name: Some(cte_name.clone()), - cte_map: bind_context.cte_context.cte_map.clone(), - }, - ..Default::default() - }; - let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, &cte.query)?; - let bind_result = CteBindResult { - s_expr, - bind_context: cte_bind_context, + let bind_result = if with.recursive { + None + } else { + let mut cte_bind_context = BindContext { + cte_context: CteContext { + cte_name: Some(cte_name.clone()), + cte_map: bind_context.cte_context.cte_map.clone(), + }, + ..Default::default() + }; + let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, &cte.query)?; + let bind_result = CteBindResult { + s_expr, + bind_context: cte_bind_context, + }; + Some(bind_result) }; let cte_info = CteInfo { @@ -262,7 +267,8 @@ impl Binder { .get(&cte_name) .unwrap() .bind_result - .clone(); + .clone() + .unwrap(); let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); current_expr = SExpr::create_binary( diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 113c33a54333e..1ce523d2e4d10 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -87,7 +87,7 @@ impl Binder { let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { if cte_info.materialized { - let mut cte_bind_context = cte_info.bind_result.bind_context.clone(); + let mut cte_bind_context = cte_info.bind_result.as_ref().unwrap().bind_context.clone(); // Apply column aliases let mut cols_alias = cte_info.columns_alias.clone(); if let Some(alias) = alias { @@ -145,7 +145,7 @@ impl Binder { let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { cte_name: table_name, cte_schema, - def: cte_info.bind_result.s_expr.clone(), + def: cte_info.bind_result.as_ref().unwrap().s_expr.clone(), }))); return Ok((s_expr, new_bind_context)); } else { From 67c2bc3950aa867ccc5408406f31c4bb56e2ee02 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 3 Jul 2025 16:18:30 +0800 Subject: [PATCH 21/37] fix --- .../sql/src/planner/binder/bind_query/bind.rs | 3 +- .../binder/bind_table_reference/bind_table.rs | 3 +- .../sql/src/planner/plans/cte_consumer.rs | 40 +++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index e2c6473bc0c0f..3295167e99f3b 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -158,7 +158,8 @@ impl Binder { }, ..Default::default() }; - let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, &cte.query)?; + let (s_expr, cte_bind_context) = + self.bind_query(&mut cte_bind_context, &cte.query)?; let bind_result = CteBindResult { s_expr, bind_context: cte_bind_context, diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 1ce523d2e4d10..fca9c63c11bb2 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -87,7 +87,8 @@ impl Binder { let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { if cte_info.materialized { - let mut cte_bind_context = cte_info.bind_result.as_ref().unwrap().bind_context.clone(); + let mut cte_bind_context = + cte_info.bind_result.as_ref().unwrap().bind_context.clone(); // Apply column aliases let mut cols_alias = cte_info.columns_alias.clone(); if let Some(alias) = alias { diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index e439b5b558636..01580bc10133c 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -16,10 +16,15 @@ use std::hash::Hash; use std::hash::Hasher; use std::sync::Arc; +use databend_common_catalog::table_context::TableContext; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; +use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RelExpr; +use crate::optimizer::ir::RelationalProperty; +use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::SExpr; use crate::optimizer::ir::StatInfo; use crate::plans::Operator; @@ -52,4 +57,39 @@ impl Operator for CTEConsumer { fn derive_stats(&self, _rel_expr: &RelExpr) -> Result> { RelExpr::with_s_expr(&self.def).derive_cardinality() } + + /// Derive relational property + fn derive_relational_prop(&self, _rel_expr: &RelExpr) -> Result> { + RelExpr::with_s_expr(&self.def).derive_relational_prop() + } + + /// Derive physical property + fn derive_physical_prop(&self, _rel_expr: &RelExpr) -> Result { + RelExpr::with_s_expr(&self.def).derive_physical_prop() + } + + /// Compute required property for child with index `child_index` + fn compute_required_prop_child( + &self, + _ctx: Arc, + _rel_expr: &RelExpr, + _child_index: usize, + _required: &RequiredProperty, + ) -> Result { + Err(ErrorCode::Internal( + "Cannot compute required property for children of cte_consumer".to_string(), + )) + } + + /// Enumerate all possible combinations of required property for children + fn compute_required_prop_children( + &self, + _ctx: Arc, + _rel_expr: &RelExpr, + _required: &RequiredProperty, + ) -> Result>> { + Err(ErrorCode::Internal( + "Cannot compute required property for children of cte_consumer".to_string(), + )) + } } From 9a8eb3bdc932e397862e3fe3a4b07c75830bfc61 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Tue, 8 Jul 2025 16:27:39 +0800 Subject: [PATCH 22/37] fix --- .../sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index 4d8c1d4daac5a..183b4db69c21e 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -275,6 +275,10 @@ impl DPhpyOptimizer { s_expr: &SExpr, ) -> Result<(Arc, bool)> { let new_s_expr = self.new_children(s_expr).await?; + self.join_relations.push(JoinRelation::new( + &new_s_expr, + self.sample_executor().clone(), + )); Ok((Arc::new(new_s_expr), true)) } From 30aa9f386e17a05b7f8aa4701f22bebacb8e460f Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 9 Jul 2025 10:26:34 +0800 Subject: [PATCH 23/37] fix --- .../optimizer/optimizers/hyper_dp/dphyp.rs | 60 ++++++++++++++++++- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index 183b4db69c21e..3db2af0c0ae53 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -51,6 +51,8 @@ pub struct DPhpyOptimizer { join_relations: Vec, // base table index -> index of join_relations table_index_map: HashMap, + // cte name -> table indexes referenced by cte + cte_table_index_map: HashMap>, dp_table: HashMap, JoinNode>, query_graph: QueryGraph, relation_set_tree: RelationSetTree, @@ -66,6 +68,7 @@ impl DPhpyOptimizer { opt_ctx, join_relations: vec![], table_index_map: Default::default(), + cte_table_index_map: Default::default(), dp_table: Default::default(), query_graph: QueryGraph::new(), relation_set_tree: Default::default(), @@ -274,7 +277,28 @@ impl DPhpyOptimizer { &mut self, s_expr: &SExpr, ) -> Result<(Arc, bool)> { - let new_s_expr = self.new_children(s_expr).await?; + let m_cte = match s_expr.plan() { + RelOperator::MaterializedCTE(cte) => cte, + _ => unreachable!(), + }; + + let mut left_dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); + let left_expr = left_dphyp.optimize_async(s_expr.child(0)?).await?; + + let mut right_dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); + right_dphyp.cte_table_index_map.insert( + m_cte.cte_name.clone(), + self.table_index_map.keys().cloned().collect(), + ); + let right_expr = right_dphyp.optimize_async(s_expr.child(1)?).await?; + + // Merge table_index_map from right child into current table_index_map + let relation_idx = self.join_relations.len() as IndexType; + for table_index in right_dphyp.table_index_map.keys() { + self.table_index_map.insert(*table_index, relation_idx); + } + + let new_s_expr = s_expr.replace_children([Arc::new(left_expr), Arc::new(right_expr)]); self.join_relations.push(JoinRelation::new( &new_s_expr, self.sample_executor().clone(), @@ -282,7 +306,35 @@ impl DPhpyOptimizer { Ok((Arc::new(new_s_expr), true)) } - async fn process_cte_consumer_node(&mut self, s_expr: &SExpr) -> Result<(Arc, bool)> { + async fn process_cte_consumer_node( + &mut self, + s_expr: &SExpr, + join_relation: Option<&SExpr>, + ) -> Result<(Arc, bool)> { + let cte_consumer = match s_expr.plan() { + RelOperator::CTEConsumer(consumer) => consumer, + _ => unreachable!(), + }; + + let join_relation = if let Some(relation) = join_relation { + // Check if relation contains filter, if exists, check if the filter in `filters` + // If exists, remove it from `filters` + self.check_filter(relation); + JoinRelation::new(relation, self.sample_executor().clone()) + } else { + JoinRelation::new(s_expr, self.sample_executor().clone()) + }; + + // Get table indexes from cte_table_index_map for this CTE + if let Some(table_indexes) = self.cte_table_index_map.get(&cte_consumer.cte_name) { + // Map each table index to current join_relations index + let relation_idx = self.join_relations.len() as IndexType; + for table_index in table_indexes { + self.table_index_map.insert(*table_index, relation_idx); + } + } + + self.join_relations.push(join_relation); Ok((Arc::new(s_expr.clone()), true)) } @@ -349,7 +401,9 @@ impl DPhpyOptimizer { RelOperator::Join(_) => self.process_join_node(s_expr, join_conditions).await, RelOperator::MaterializedCTE(_) => self.process_materialized_cte_node(s_expr).await, - RelOperator::CTEConsumer(_) => self.process_cte_consumer_node(s_expr).await, + RelOperator::CTEConsumer(_) => { + self.process_cte_consumer_node(s_expr, join_relation).await + } RelOperator::ProjectSet(_) | RelOperator::Aggregate(_) From e39af8e7a4fea922ab5ebd7db30e50d8bbdc5f92 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 9 Jul 2025 10:57:39 +0800 Subject: [PATCH 24/37] fix --- .../optimizer/optimizers/hyper_dp/dphyp.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index 3db2af0c0ae53..9ca9bb4cdcfc3 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -94,8 +94,10 @@ impl DPhpyOptimizer { // Parallel process children: start a new dphyp for each child. let left_expr = s_expr.children[0].clone(); let opt_ctx = self.opt_ctx.clone(); + let cte_table_index_map = self.cte_table_index_map.clone(); let left_res = spawn(async move { let mut dphyp = DPhpyOptimizer::new(opt_ctx.clone()); + dphyp.cte_table_index_map = cte_table_index_map; ( dphyp.optimize_async(&left_expr).await, dphyp.table_index_map, @@ -104,8 +106,10 @@ impl DPhpyOptimizer { let right_expr = s_expr.children[1].clone(); let opt_ctx = self.opt_ctx.clone(); + let cte_table_index_map = self.cte_table_index_map.clone(); let right_res = spawn(async move { let mut dphyp = DPhpyOptimizer::new(opt_ctx.clone()); + dphyp.cte_table_index_map = cte_table_index_map; ( dphyp.optimize_async(&right_expr).await, dphyp.table_index_map, @@ -137,6 +141,7 @@ impl DPhpyOptimizer { /// Process a subquery expression async fn process_subquery(&mut self, s_expr: &SExpr) -> Result<(Arc, bool)> { let mut dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); + dphyp.cte_table_index_map = self.cte_table_index_map.clone(); let new_s_expr = Arc::new(dphyp.optimize_async(s_expr).await?); // Merge `table_index_map` of subquery into current `table_index_map`. @@ -283,12 +288,15 @@ impl DPhpyOptimizer { }; let mut left_dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); + left_dphyp.cte_table_index_map = self.cte_table_index_map.clone(); let left_expr = left_dphyp.optimize_async(s_expr.child(0)?).await?; let mut right_dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); + right_dphyp.cte_table_index_map = self.cte_table_index_map.clone(); + // Use left_dphyp's table_index_map for the CTE definition right_dphyp.cte_table_index_map.insert( m_cte.cte_name.clone(), - self.table_index_map.keys().cloned().collect(), + left_dphyp.table_index_map.keys().cloned().collect(), ); let right_expr = right_dphyp.optimize_async(s_expr.child(1)?).await?; @@ -325,10 +333,9 @@ impl DPhpyOptimizer { JoinRelation::new(s_expr, self.sample_executor().clone()) }; - // Get table indexes from cte_table_index_map for this CTE + // Map table indexes before adding to join_relations + let relation_idx = self.join_relations.len() as IndexType; if let Some(table_indexes) = self.cte_table_index_map.get(&cte_consumer.cte_name) { - // Map each table index to current join_relations index - let relation_idx = self.join_relations.len() as IndexType; for table_index in table_indexes { self.table_index_map.insert(*table_index, relation_idx); } From a39a569c885599e68a3403d7dc0c56d980dafb07 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 9 Jul 2025 12:04:52 +0800 Subject: [PATCH 25/37] CleanupUnusedCTE --- .../sql/src/planner/optimizer/optimizer.rs | 5 +- .../src/planner/optimizer/optimizers/mod.rs | 1 + .../operator/cte/cleanup_unused_cte.rs | 93 +++++++++++++++ .../optimizer/optimizers/operator/cte/mod.rs | 17 +++ .../optimizer/optimizers/operator/mod.rs | 2 + .../suites/query/cleanup_unused_cte.test | 112 ++++++++++++++++++ 6 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs create mode 100644 src/query/sql/src/planner/optimizer/optimizers/operator/cte/mod.rs create mode 100644 tests/sqllogictests/suites/query/cleanup_unused_cte.test diff --git a/src/query/sql/src/planner/optimizer/optimizer.rs b/src/query/sql/src/planner/optimizer/optimizer.rs index 73213ecd31832..2141a9e9bbe0c 100644 --- a/src/query/sql/src/planner/optimizer/optimizer.rs +++ b/src/query/sql/src/planner/optimizer/optimizer.rs @@ -26,6 +26,7 @@ use crate::binder::MutationType; use crate::optimizer::ir::Memo; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::distributed::BroadcastToShuffleOptimizer; +use crate::optimizer::optimizers::operator::CleanupUnusedCTEOptimizer; use crate::optimizer::optimizers::operator::DeduplicateJoinConditionOptimizer; use crate::optimizer::optimizers::operator::PullUpFilterOptimizer; use crate::optimizer::optimizers::operator::RuleNormalizeAggregateOptimizer; @@ -273,7 +274,9 @@ pub async fn optimize_query(opt_ctx: Arc, s_expr: SExpr) -> Re .add_if( !opt_ctx.get_planning_agg_index(), RecursiveRuleOptimizer::new(opt_ctx.clone(), [RuleID::EliminateEvalScalar].as_slice()), - ); + ) + // 15. Clean up unused CTEs + .add(CleanupUnusedCTEOptimizer); // 15. Execute the pipeline let s_expr = pipeline.execute().await?; diff --git a/src/query/sql/src/planner/optimizer/optimizers/mod.rs b/src/query/sql/src/planner/optimizer/optimizers/mod.rs index c4f5e6998b631..aa38f80db04fc 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/mod.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/mod.rs @@ -21,3 +21,4 @@ pub mod rule; pub use cascades::CascadesOptimizer; pub use hyper_dp::DPhpyOptimizer; +pub use operator::CleanupUnusedCTEOptimizer; diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs new file mode 100644 index 0000000000000..b29d06afe460d --- /dev/null +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs @@ -0,0 +1,93 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashSet; +use std::sync::Arc; + +use databend_common_exception::Result; + +use crate::optimizer::ir::SExpr; +use crate::optimizer::Optimizer; +use crate::plans::RelOperator; + +/// Optimizer that removes unused CTEs from the query plan. +/// This optimizer should be applied at the end of the optimization pipeline +/// to clean up any CTEs that are not referenced by any CTEConsumer. +pub struct CleanupUnusedCTEOptimizer; + +impl CleanupUnusedCTEOptimizer { + /// Collect all CTE names that are referenced by CTEConsumer nodes + fn collect_referenced_ctes(s_expr: &SExpr) -> Result> { + let mut referenced_ctes = HashSet::new(); + Self::collect_referenced_ctes_recursive(s_expr, &mut referenced_ctes)?; + Ok(referenced_ctes) + } + + /// Recursively traverse the expression tree to find CTEConsumer nodes + fn collect_referenced_ctes_recursive( + s_expr: &SExpr, + referenced_ctes: &mut HashSet, + ) -> Result<()> { + // Check if current node is a CTEConsumer + if let RelOperator::CTEConsumer(consumer) = s_expr.plan() { + referenced_ctes.insert(consumer.cte_name.clone()); + } + + // Recursively process children + for child in s_expr.children() { + Self::collect_referenced_ctes_recursive(child, referenced_ctes)?; + } + + Ok(()) + } + + /// Remove unused CTEs from the expression tree + fn remove_unused_ctes(s_expr: &SExpr, referenced_ctes: &HashSet) -> Result { + if let RelOperator::MaterializedCTE(m_cte) = s_expr.plan() { + // If this CTE is not referenced, remove it by returning the right child + if !referenced_ctes.contains(&m_cte.cte_name) { + // Return the right child (main query) and skip the left child (CTE definition) + let right_child = s_expr.child(1)?; + return Self::remove_unused_ctes(right_child, referenced_ctes); + } + } + + // Process children recursively + let mut optimized_children = Vec::with_capacity(s_expr.arity()); + for child in s_expr.children() { + let optimized_child = Self::remove_unused_ctes(child, referenced_ctes)?; + optimized_children.push(Arc::new(optimized_child)); + } + + // Create new expression with optimized children + let mut new_expr = s_expr.clone(); + new_expr.children = optimized_children; + Ok(new_expr) + } +} + +#[async_trait::async_trait] +impl Optimizer for CleanupUnusedCTEOptimizer { + fn name(&self) -> String { + "CleanupUnusedCTEOptimizer".to_string() + } + + async fn optimize(&mut self, s_expr: &SExpr) -> Result { + // Collect all referenced CTEs + let referenced_ctes = Self::collect_referenced_ctes(s_expr)?; + + // Remove unused CTEs + Self::remove_unused_ctes(s_expr, &referenced_ctes) + } +} diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/mod.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/mod.rs new file mode 100644 index 0000000000000..0aeab30d3574b --- /dev/null +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/mod.rs @@ -0,0 +1,17 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod cleanup_unused_cte; + +pub use cleanup_unused_cte::CleanupUnusedCTEOptimizer; diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/mod.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/mod.rs index 0c0566a886f9e..93c8833bec214 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/mod.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/mod.rs @@ -13,12 +13,14 @@ // limitations under the License. mod aggregate; +mod cte; mod decorrelate; mod filter; mod join; pub use aggregate::RuleNormalizeAggregateOptimizer; pub use aggregate::RuleStatsAggregateOptimizer; +pub use cte::CleanupUnusedCTEOptimizer; pub use decorrelate::FlattenInfo; pub use decorrelate::SubqueryDecorrelatorOptimizer; pub use decorrelate::UnnestResult; diff --git a/tests/sqllogictests/suites/query/cleanup_unused_cte.test b/tests/sqllogictests/suites/query/cleanup_unused_cte.test new file mode 100644 index 0000000000000..4c38d8a4e6845 --- /dev/null +++ b/tests/sqllogictests/suites/query/cleanup_unused_cte.test @@ -0,0 +1,112 @@ +# Test for CleanupUnusedCTEOptimizer +# This test verifies that unused CTEs are properly removed from the query plan + +# Test case 1: CTE is used, should not be removed +query I +with t1 as materialized (select number as a from numbers(10)) select t1.a from t1; +---- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +# Test case 2: CTE is not used, should be removed +query I +with t1 as materialized (select number as a from numbers(10)) select number as b from numbers(5); +---- +0 +1 +2 +3 +4 + +# Test case 3: Multiple CTEs, some used and some unused +query I +with t1 as materialized (select number as a from numbers(10)), + t2 as materialized (select number as b from numbers(20)), + t3 as materialized (select number as c from numbers(30)) +select t1.a from t1 join t2 on t1.a = t2.b; +---- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +# Test case 4: Nested CTEs, inner CTE is unused +query I +with t1 as materialized (select number as a from numbers(10)), + t2 as materialized (select a as b from t1), + t3 as materialized (select number as c from numbers(5)) +select t2.b from t2; +---- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +# Test case 5: All CTEs are unused +query I +with t1 as materialized (select number as a from numbers(10)), + t2 as materialized (select number as b from numbers(20)) +select number as c from numbers(3); +---- +0 +1 +2 + +# Test case 6: CTE with complex query, should be removed when unused +query I +with t1 as materialized ( + select number as a, number * 2 as b + from numbers(10) + where number > 5 +) +select number as c from numbers(3); +---- +0 +1 +2 + +# Test case 7: CTE with aggregation, should be removed when unused +query I +with t1 as materialized ( + select number as a, count(*) as cnt + from numbers(10) + group by number +) +select number as b from numbers(3); +---- +0 +1 +2 + +# Test case 8: CTE with join, should be removed when unused +query I +with t1 as materialized ( + select n1.number as a, n2.number as b + from numbers(5) n1 + join numbers(5) n2 on n1.number = n2.number +) +select number as c from numbers(3); +---- +0 +1 +2 \ No newline at end of file From 36866865f0e31ff8fbca0d13f679c2430b4b8a12 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 14:41:09 +0800 Subject: [PATCH 26/37] fix --- .../binder/bind_table_reference/bind_cte.rs | 132 ++++++++++++++++++ .../binder/bind_table_reference/bind_table.rs | 70 +--------- .../binder/bind_table_reference/mod.rs | 1 + 3 files changed, 134 insertions(+), 69 deletions(-) create mode 100644 src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs new file mode 100644 index 0000000000000..9a066ed2617aa --- /dev/null +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -0,0 +1,132 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_ast::ast::Query; +use databend_common_ast::ast::TableAlias; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_expression::DataField; +use databend_common_expression::DataSchemaRefExt; +use indexmap::IndexMap; + +use crate::binder::BindContext; +use crate::binder::Binder; +use crate::binder::CteContext; +use crate::binder::CteInfo; +use crate::normalize_identifier; +use crate::optimizer::ir::SExpr; +use crate::plans::CTEConsumer; +use crate::plans::RelOperator; + +impl Binder { + pub fn bind_cte_consumer( + &mut self, + bind_context: &mut BindContext, + table_name: &str, + alias: &Option, + cte_info: &CteInfo, + ) -> Result<(SExpr, BindContext)> { + let (_, cte_bind_context) = self.bind_cte_definition( + table_name, + bind_context.cte_context.cte_map.as_ref(), + &cte_info.query, + )?; + + let (table_alias, column_alias) = match alias { + Some(alias) => { + let table_alias = normalize_identifier(&alias.name, &self.name_resolution_ctx).name; + let column_alias = if alias.columns.is_empty() { + cte_info.columns_alias.clone() + } else { + alias + .columns + .iter() + .map(|column| normalize_identifier(column, &self.name_resolution_ctx).name) + .collect() + }; + (table_alias, column_alias) + } + None => (table_name.to_string(), cte_info.columns_alias.clone()), + }; + + if !column_alias.is_empty() && column_alias.len() != cte_bind_context.columns.len() { + return Err(ErrorCode::SemanticError(format!( + "The CTE '{}' has {} columns ({:?}), but {} aliases ({:?}) were provided. Ensure the number of aliases matches the number of columns in the CTE.", + table_name, + cte_bind_context.columns.len(), + cte_bind_context.columns.iter().map(|c| &c.column_name).collect::>(), + column_alias.len(), + column_alias, + ))); + } + + let mut cte_output_columns = cte_bind_context.columns.clone(); + for column in cte_output_columns.iter_mut() { + column.database_name = None; + column.table_name = Some(table_alias.clone()); + } + for (index, column_name) in column_alias.iter().enumerate() { + cte_output_columns[index].column_name = column_name.clone(); + } + + let fields = cte_output_columns + .iter() + .map(|column_binding| { + DataField::new( + &column_binding.index.to_string(), + *column_binding.data_type.clone(), + ) + }) + .collect(); + let cte_schema = DataSchemaRefExt::create(fields); + + let mut new_bind_context = bind_context.clone(); + for column in cte_output_columns { + new_bind_context.add_column_binding(column); + } + + let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { + cte_name: table_name.to_string(), + cte_schema, + def: cte_info.bind_result.as_ref().unwrap().s_expr.clone(), + }))); + Ok((s_expr, new_bind_context)) + } + + fn bind_cte_definition( + &mut self, + cte_name: &str, + cte_map: &IndexMap, + query: &Query, + ) -> Result<(SExpr, BindContext)> { + let mut prev_cte_map = Box::new(IndexMap::new()); + for (name, cte_info) in cte_map.iter() { + if name == cte_name { + break; + } + prev_cte_map.insert(name.clone(), cte_info.clone()); + } + let mut cte_bind_context = BindContext { + cte_context: CteContext { + cte_name: Some(cte_name.to_string()), + cte_map: prev_cte_map, + }, + ..Default::default() + }; + let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, query)?; + Ok((s_expr, cte_bind_context)) + } +} diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index fca9c63c11bb2..d43bc3f7dfbba 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Arc; - use databend_common_ast::ast::Identifier; use databend_common_ast::ast::SampleConfig; use databend_common_ast::ast::Statement; @@ -29,17 +27,12 @@ use databend_common_catalog::table_with_options::get_with_opt_consume; use databend_common_catalog::table_with_options::get_with_opt_max_batch_size; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::DataField; -use databend_common_expression::DataSchemaRefExt; use databend_common_storages_view::view_table::QUERY; use databend_storages_common_table_meta::table::get_change_type; use crate::binder::util::TableIdentifier; use crate::binder::Binder; -use crate::normalize_identifier; use crate::optimizer::ir::SExpr; -use crate::plans::CTEConsumer; -use crate::plans::RelOperator; use crate::BindContext; impl Binder { /// Bind a base table. @@ -87,68 +80,7 @@ impl Binder { let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { if cte_info.materialized { - let mut cte_bind_context = - cte_info.bind_result.as_ref().unwrap().bind_context.clone(); - // Apply column aliases - let mut cols_alias = cte_info.columns_alias.clone(); - if let Some(alias) = alias { - for (idx, col_alias) in alias.columns.iter().enumerate() { - if idx < cte_info.columns_alias.len() { - cols_alias[idx] = col_alias.name.clone(); - } else { - cols_alias.push(col_alias.name.clone()); - } - } - } - - let alias_table_name = alias - .as_ref() - .map(|alias| normalize_identifier(&alias.name, &self.name_resolution_ctx).name) - .unwrap_or_else(|| table_name.to_string()); - - for column in cte_bind_context.columns.iter_mut() { - column.database_name = None; - column.table_name = Some(alias_table_name.clone()); - } - - if cols_alias.len() > cte_bind_context.columns.len() { - return Err(ErrorCode::SemanticError(format!( - "The CTE '{}' has {} columns, but {} aliases were provided. Ensure the number of aliases matches the number of columns in the CTE.", - table_name, - cte_bind_context.columns.len(), - cols_alias.len() - )) - .set_span(*span)); - } - - for (index, column_name) in cols_alias.iter().enumerate() { - cte_bind_context.columns[index].column_name = column_name.clone(); - } - - let fields = cte_bind_context - .columns - .iter() - .map(|column_binding| { - DataField::new( - &column_binding.index.to_string(), - *column_binding.data_type.clone(), - ) - }) - .collect(); - let cte_schema = DataSchemaRefExt::create(fields); - - // Add the columns to the new bind context - let mut new_bind_context = bind_context.clone(); - for column in cte_bind_context.columns { - new_bind_context.add_column_binding(column); - } - - let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { - cte_name: table_name, - cte_schema, - def: cte_info.bind_result.as_ref().unwrap().s_expr.clone(), - }))); - return Ok((s_expr, new_bind_context)); + return self.bind_cte_consumer(bind_context, &table_name, alias, cte_info); } else { if self .metadata diff --git a/src/query/sql/src/planner/binder/bind_table_reference/mod.rs b/src/query/sql/src/planner/binder/bind_table_reference/mod.rs index 542911474aa34..75a42e6f60ddf 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/mod.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/mod.rs @@ -13,6 +13,7 @@ // limitations under the License. mod bind; +mod bind_cte; mod bind_join; mod bind_location; mod bind_obfuscate; From fb7bfbd9d9e1d34892d5f019d612c359d5533f04 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 15:30:56 +0800 Subject: [PATCH 27/37] fix --- .../sql/src/planner/binder/bind_table_reference/bind_cte.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index 9a066ed2617aa..c009c786ff707 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -39,7 +39,7 @@ impl Binder { alias: &Option, cte_info: &CteInfo, ) -> Result<(SExpr, BindContext)> { - let (_, cte_bind_context) = self.bind_cte_definition( + let (s_expr, cte_bind_context) = self.bind_cte_definition( table_name, bind_context.cte_context.cte_map.as_ref(), &cte_info.query, @@ -101,7 +101,7 @@ impl Binder { let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { cte_name: table_name.to_string(), cte_schema, - def: cte_info.bind_result.as_ref().unwrap().s_expr.clone(), + def: s_expr, }))); Ok((s_expr, new_bind_context)) } From e600f31a39d6347bda7e1613626785c4d7b67e73 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 15:46:08 +0800 Subject: [PATCH 28/37] fix --- .../optimizer/optimizers/hyper_dp/dphyp.rs | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index 9ca9bb4cdcfc3..d4a665be1b700 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -51,8 +51,6 @@ pub struct DPhpyOptimizer { join_relations: Vec, // base table index -> index of join_relations table_index_map: HashMap, - // cte name -> table indexes referenced by cte - cte_table_index_map: HashMap>, dp_table: HashMap, JoinNode>, query_graph: QueryGraph, relation_set_tree: RelationSetTree, @@ -68,7 +66,6 @@ impl DPhpyOptimizer { opt_ctx, join_relations: vec![], table_index_map: Default::default(), - cte_table_index_map: Default::default(), dp_table: Default::default(), query_graph: QueryGraph::new(), relation_set_tree: Default::default(), @@ -94,10 +91,8 @@ impl DPhpyOptimizer { // Parallel process children: start a new dphyp for each child. let left_expr = s_expr.children[0].clone(); let opt_ctx = self.opt_ctx.clone(); - let cte_table_index_map = self.cte_table_index_map.clone(); let left_res = spawn(async move { let mut dphyp = DPhpyOptimizer::new(opt_ctx.clone()); - dphyp.cte_table_index_map = cte_table_index_map; ( dphyp.optimize_async(&left_expr).await, dphyp.table_index_map, @@ -106,10 +101,8 @@ impl DPhpyOptimizer { let right_expr = s_expr.children[1].clone(); let opt_ctx = self.opt_ctx.clone(); - let cte_table_index_map = self.cte_table_index_map.clone(); let right_res = spawn(async move { let mut dphyp = DPhpyOptimizer::new(opt_ctx.clone()); - dphyp.cte_table_index_map = cte_table_index_map; ( dphyp.optimize_async(&right_expr).await, dphyp.table_index_map, @@ -141,7 +134,6 @@ impl DPhpyOptimizer { /// Process a subquery expression async fn process_subquery(&mut self, s_expr: &SExpr) -> Result<(Arc, bool)> { let mut dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); - dphyp.cte_table_index_map = self.cte_table_index_map.clone(); let new_s_expr = Arc::new(dphyp.optimize_async(s_expr).await?); // Merge `table_index_map` of subquery into current `table_index_map`. @@ -282,22 +274,10 @@ impl DPhpyOptimizer { &mut self, s_expr: &SExpr, ) -> Result<(Arc, bool)> { - let m_cte = match s_expr.plan() { - RelOperator::MaterializedCTE(cte) => cte, - _ => unreachable!(), - }; - let mut left_dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); - left_dphyp.cte_table_index_map = self.cte_table_index_map.clone(); let left_expr = left_dphyp.optimize_async(s_expr.child(0)?).await?; let mut right_dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); - right_dphyp.cte_table_index_map = self.cte_table_index_map.clone(); - // Use left_dphyp's table_index_map for the CTE definition - right_dphyp.cte_table_index_map.insert( - m_cte.cte_name.clone(), - left_dphyp.table_index_map.keys().cloned().collect(), - ); let right_expr = right_dphyp.optimize_async(s_expr.child(1)?).await?; // Merge table_index_map from right child into current table_index_map @@ -335,16 +315,31 @@ impl DPhpyOptimizer { // Map table indexes before adding to join_relations let relation_idx = self.join_relations.len() as IndexType; - if let Some(table_indexes) = self.cte_table_index_map.get(&cte_consumer.cte_name) { - for table_index in table_indexes { - self.table_index_map.insert(*table_index, relation_idx); - } + let table_indexes = Self::get_table_indexes(&cte_consumer.def); + for table_index in table_indexes { + self.table_index_map.insert(table_index, relation_idx); } self.join_relations.push(join_relation); Ok((Arc::new(s_expr.clone()), true)) } + fn get_table_indexes(s_expr: &SExpr) -> Vec { + let mut table_indexes = Vec::new(); + Self::collect_table_indexes_recursive(s_expr, &mut table_indexes); + table_indexes + } + + fn collect_table_indexes_recursive(s_expr: &SExpr, table_indexes: &mut Vec) { + if let RelOperator::Scan(scan) = s_expr.plan() { + table_indexes.push(scan.table_index); + } + + for child in s_expr.children() { + Self::collect_table_indexes_recursive(child, table_indexes); + } + } + /// Process a unary operator node async fn process_unary_node( &mut self, From 5e0752a7cbcb0fa466fc9489c5e8b5ac30228191 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 16:10:40 +0800 Subject: [PATCH 29/37] fix --- .../sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index d4a665be1b700..bb17f389104a0 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -335,6 +335,10 @@ impl DPhpyOptimizer { table_indexes.push(scan.table_index); } + if let RelOperator::CTEConsumer(cte_consumer) = s_expr.plan() { + Self::collect_table_indexes_recursive(&cte_consumer.def, table_indexes); + } + for child in s_expr.children() { Self::collect_table_indexes_recursive(child, table_indexes); } From 4606b12d7e36b0b8592edc7686d75acccdbfc6d9 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 19:12:23 +0800 Subject: [PATCH 30/37] refine --- .../sql/src/planner/binder/bind_context.rs | 1 - .../planner/binder/bind_mutation/delete.rs | 2 +- .../planner/binder/bind_mutation/update.rs | 2 +- .../sql/src/planner/binder/bind_query/bind.rs | 96 +------------------ .../binder/bind_table_reference/bind_cte.rs | 61 +++++++++++- .../src/planner/binder/copy_into_location.rs | 2 +- .../sql/src/planner/binder/copy_into_table.rs | 2 +- src/query/sql/src/planner/binder/insert.rs | 2 +- 8 files changed, 66 insertions(+), 102 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 790ab40698d8f..3a0d4179e77c4 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -195,7 +195,6 @@ impl CteContext { pub struct CteInfo { pub columns_alias: Vec, pub query: Query, - pub bind_result: Option, pub materialized: bool, pub recursive: bool, pub columns: Vec, diff --git a/src/query/sql/src/planner/binder/bind_mutation/delete.rs b/src/query/sql/src/planner/binder/bind_mutation/delete.rs index b1b0f07e74529..0e8fcb8d7d671 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/delete.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/delete.rs @@ -41,7 +41,7 @@ impl Binder { .. } = stamt; - self.bind_cte_def(bind_context, with)?; + self.init_cte(bind_context, with)?; let target_table_identifier = if let TableReference::Table { catalog, diff --git a/src/query/sql/src/planner/binder/bind_mutation/update.rs b/src/query/sql/src/planner/binder/bind_mutation/update.rs index c267af8dc25a5..56de12c2c2921 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/update.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/update.rs @@ -63,7 +63,7 @@ impl Binder { .. } = stmt; - self.bind_cte_def(bind_context, with)?; + self.init_cte(bind_context, with)?; let target_table_identifier = TableIdentifier::new(self, catalog, database, table, table_alias); diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 3295167e99f3b..315dbad48db93 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -25,16 +25,12 @@ use databend_common_exception::Result; use derive_visitor::Drive; use derive_visitor::Visitor; -use crate::binder::CteBindResult; -use crate::binder::CteInfo; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::planner::binder::scalar::ScalarBinder; use crate::planner::binder::BindContext; use crate::planner::binder::Binder; -use crate::planner::binder::CteContext; use crate::plans::BoundColumnRef; -use crate::plans::MaterializedCTE; use crate::plans::ScalarExpr; use crate::plans::Sort; use crate::plans::SortItem; @@ -74,7 +70,7 @@ impl Binder { } // Bind cte definition. - self.bind_cte_def(bind_context, &with)?; + self.init_cte(bind_context, &with)?; // Extract limit and offset from query. let (limit, offset) = self.extract_limit_and_offset(query)?; @@ -125,62 +121,6 @@ impl Binder { Ok(()) } - pub(crate) fn bind_cte_def( - &mut self, - bind_context: &mut BindContext, - with: &Option, - ) -> Result<()> { - let Some(with) = with else { - return Ok(()); - }; - for cte in with.ctes.iter() { - let cte_name = self.normalize_identifier(&cte.alias.name).name; - if bind_context.cte_context.cte_map.contains_key(&cte_name) { - return Err(ErrorCode::SemanticError(format!( - "Duplicate common table expression: {cte_name}" - ))); - } - - let column_name = cte - .alias - .columns - .iter() - .map(|ident| self.normalize_identifier(ident).name) - .collect(); - - let bind_result = if with.recursive { - None - } else { - let mut cte_bind_context = BindContext { - cte_context: CteContext { - cte_name: Some(cte_name.clone()), - cte_map: bind_context.cte_context.cte_map.clone(), - }, - ..Default::default() - }; - let (s_expr, cte_bind_context) = - self.bind_query(&mut cte_bind_context, &cte.query)?; - let bind_result = CteBindResult { - s_expr, - bind_context: cte_bind_context, - }; - Some(bind_result) - }; - - let cte_info = CteInfo { - columns_alias: column_name, - query: *cte.query.clone(), - recursive: with.recursive, - materialized: cte.materialized, - columns: vec![], - bind_result, - }; - bind_context.cte_context.cte_map.insert(cte_name, cte_info); - } - - Ok(()) - } - pub(crate) fn bind_query_order_by( &mut self, bind_context: &mut BindContext, @@ -248,38 +188,4 @@ impl Binder { Arc::new(child), )) } - - fn bind_materialized_cte( - &mut self, - with: &With, - main_query_expr: SExpr, - cte_context: CteContext, - ) -> Result { - let mut current_expr = main_query_expr; - - for cte in with.ctes.iter().rev() { - if cte.materialized { - let cte_name = self.normalize_identifier(&cte.alias.name).name; - let CteBindResult { - s_expr, - bind_context, - } = cte_context - .cte_map - .get(&cte_name) - .unwrap() - .bind_result - .clone() - .unwrap(); - - let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); - current_expr = SExpr::create_binary( - Arc::new(materialized_cte.into()), - Arc::new(s_expr), - Arc::new(current_expr), - ); - } - } - - Ok(current_expr) - } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index c009c786ff707..925117e218436 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use databend_common_ast::ast::Query; use databend_common_ast::ast::TableAlias; +use databend_common_ast::ast::With; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataField; @@ -29,9 +30,41 @@ use crate::binder::CteInfo; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::plans::CTEConsumer; +use crate::plans::MaterializedCTE; use crate::plans::RelOperator; impl Binder { + pub fn init_cte(&mut self, bind_context: &mut BindContext, with: &Option) -> Result<()> { + let Some(with) = with else { + return Ok(()); + }; + for cte in with.ctes.iter() { + let cte_name = self.normalize_identifier(&cte.alias.name).name; + if bind_context.cte_context.cte_map.contains_key(&cte_name) { + return Err(ErrorCode::SemanticError(format!( + "Duplicate common table expression: {cte_name}" + ))); + } + + let column_name = cte + .alias + .columns + .iter() + .map(|ident| self.normalize_identifier(ident).name) + .collect(); + + let cte_info = CteInfo { + columns_alias: column_name, + query: *cte.query.clone(), + recursive: with.recursive, + materialized: cte.materialized, + columns: vec![], + }; + bind_context.cte_context.cte_map.insert(cte_name, cte_info); + } + + Ok(()) + } pub fn bind_cte_consumer( &mut self, bind_context: &mut BindContext, @@ -106,7 +139,7 @@ impl Binder { Ok((s_expr, new_bind_context)) } - fn bind_cte_definition( + pub fn bind_cte_definition( &mut self, cte_name: &str, cte_map: &IndexMap, @@ -129,4 +162,30 @@ impl Binder { let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, query)?; Ok((s_expr, cte_bind_context)) } + + pub fn bind_materialized_cte( + &mut self, + with: &With, + main_query_expr: SExpr, + cte_context: CteContext, + ) -> Result { + let mut current_expr = main_query_expr; + + for cte in with.ctes.iter().rev() { + if cte.materialized { + let cte_name = self.normalize_identifier(&cte.alias.name).name; + let (s_expr, bind_context) = + self.bind_cte_definition(&cte_name, &cte_context.cte_map, &cte.query)?; + + let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); + current_expr = SExpr::create_binary( + Arc::new(materialized_cte.into()), + Arc::new(s_expr), + Arc::new(current_expr), + ); + } + } + + Ok(current_expr) + } } diff --git a/src/query/sql/src/planner/binder/copy_into_location.rs b/src/query/sql/src/planner/binder/copy_into_location.rs index 01171a5102ea0..a6dd9e8d83c7a 100644 --- a/src/query/sql/src/planner/binder/copy_into_location.rs +++ b/src/query/sql/src/planner/binder/copy_into_location.rs @@ -105,7 +105,7 @@ impl Binder { } } CopyIntoLocationSource::Query(query) => { - self.bind_cte_def(bind_context, &stmt.with)?; + self.init_cte(bind_context, &stmt.with)?; self.bind_statement(bind_context, &Statement::Query(query.clone())) .await } diff --git a/src/query/sql/src/planner/binder/copy_into_table.rs b/src/query/sql/src/planner/binder/copy_into_table.rs index d10dea193440c..4c2652759705b 100644 --- a/src/query/sql/src/planner/binder/copy_into_table.rs +++ b/src/query/sql/src/planner/binder/copy_into_table.rs @@ -99,7 +99,7 @@ impl Binder { .await } CopyIntoTableSource::Query(query) => { - self.bind_cte_def(bind_context, &stmt.with)?; + self.init_cte(bind_context, &stmt.with)?; let mut max_column_position = MaxColumnPosition::new(); query.drive(&mut max_column_position); diff --git a/src/query/sql/src/planner/binder/insert.rs b/src/query/sql/src/planner/binder/insert.rs index aa374835cbb82..9cbfd4dc9b7df 100644 --- a/src/query/sql/src/planner/binder/insert.rs +++ b/src/query/sql/src/planner/binder/insert.rs @@ -111,7 +111,7 @@ impl Binder { .. } = stmt; - self.bind_cte_def(bind_context, with)?; + self.init_cte(bind_context, with)?; let table_identifier = TableIdentifier::new(self, catalog, database, table, &None); let (catalog_name, database_name, table_name) = ( From 70df77de3372f966e6bcd3296ff3cc226da1d545 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 19:14:33 +0800 Subject: [PATCH 31/37] refine --- src/query/sql/src/planner/binder/bind_context.rs | 6 ------ src/query/sql/src/planner/binder/bind_query/bind.rs | 1 - 2 files changed, 7 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 3a0d4179e77c4..758eae2c15169 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -200,12 +200,6 @@ pub struct CteInfo { pub columns: Vec, } -#[derive(Clone, Debug)] -pub struct CteBindResult { - pub s_expr: SExpr, - pub bind_context: BindContext, -} - impl BindContext { pub fn new() -> Self { Self { diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 315dbad48db93..9090af1ae1bd3 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -69,7 +69,6 @@ impl Binder { } } - // Bind cte definition. self.init_cte(bind_context, &with)?; // Extract limit and offset from query. From e38c70fd3be919a7ecc1034c34f2d814b5d52811 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 20:20:34 +0800 Subject: [PATCH 32/37] make lint --- src/query/sql/src/planner/binder/bind_context.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 758eae2c15169..b0c1c7a7d6608 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -42,7 +42,6 @@ use crate::binder::project_set::SetReturningInfo; use crate::binder::window::WindowInfo; use crate::binder::ColumnBindingBuilder; use crate::normalize_identifier; -use crate::optimizer::ir::SExpr; use crate::plans::ScalarExpr; use crate::ColumnSet; use crate::IndexType; From 6829408a52fd2f1f22f11981401a7132be301e10 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Fri, 11 Jul 2025 04:43:22 +0800 Subject: [PATCH 33/37] fix --- .../pipelines/builders/builder_materialized_cte.rs | 8 ++++++++ .../sql/src/executor/physical_plan_visitor.rs | 1 + .../physical_plans/physical_materialized_cte.rs | 14 ++++++++++++-- .../binder/bind_table_reference/bind_cte.rs | 2 +- .../sql/src/planner/plans/materialized_cte.rs | 11 +++++++---- 5 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 4336c2691920e..e90efe0b06163 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -32,6 +32,14 @@ impl PipelineBuilder { // build cte pipeline let mut build_res = sub_builder.finalize(&cte.left)?; + let input_schema = cte.left.output_schema()?; + Self::build_result_projection( + &self.func_ctx, + input_schema, + &cte.cte_output_columns, + &mut build_res.main_pipeline, + false, + )?; build_res.main_pipeline.try_resize(1)?; let (tx, rx) = tokio::sync::watch::channel(Arc::new(MaterializedCteData::default())); self.cte_receivers.insert(cte.cte_name.clone(), rx); diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index cedc2e63a75d0..a98cedc17f71a 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -657,6 +657,7 @@ pub trait PhysicalPlanReplacer { right: Box::new(right), stat_info: plan.stat_info.clone(), cte_name: plan.cte_name.clone(), + cte_output_columns: plan.cte_output_columns.clone(), }))) } diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index 401278e26280d..fc61ae7ab3bea 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -19,6 +19,7 @@ use crate::executor::explain::PlanStatsInfo; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; +use crate::ColumnBinding; use crate::ColumnSet; /// This is a binary operator that executes its children in order (left to right), and returns the results of the right child @@ -31,6 +32,7 @@ pub struct MaterializedCTE { pub left: Box, pub right: Box, pub cte_name: String, + pub cte_output_columns: Vec, } impl MaterializedCTE { @@ -48,8 +50,15 @@ impl PhysicalPlanBuilder { required: ColumnSet, ) -> Result { let left_side = Box::new( - self.build(s_expr.child(0)?, materialized_cte.required.clone()) - .await?, + self.build( + s_expr.child(0)?, + materialized_cte + .cte_output_columns + .iter() + .map(|c| c.index) + .collect(), + ) + .await?, ); let right_side = Box::new(self.build(s_expr.child(1)?, required).await?); Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { @@ -58,6 +67,7 @@ impl PhysicalPlanBuilder { left: left_side, right: right_side, cte_name: materialized_cte.cte_name.clone(), + cte_output_columns: materialized_cte.cte_output_columns.clone(), }))) } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index 925117e218436..e4d067cd0d40f 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -177,7 +177,7 @@ impl Binder { let (s_expr, bind_context) = self.bind_cte_definition(&cte_name, &cte_context.cte_map, &cte.query)?; - let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); + let materialized_cte = MaterializedCTE::new(cte_name, bind_context.columns); current_expr = SExpr::create_binary( Arc::new(materialized_cte.into()), Arc::new(s_expr), diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index dba74c7f23d49..523f05afb0abc 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -23,16 +23,19 @@ use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ColumnSet; +use crate::ColumnBinding; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct MaterializedCTE { pub cte_name: String, - pub required: ColumnSet, + pub cte_output_columns: Vec, } impl MaterializedCTE { - pub fn new(cte_name: String, required: ColumnSet) -> Self { - Self { cte_name, required } + pub fn new(cte_name: String, output_columns: Vec) -> Self { + Self { + cte_name, + cte_output_columns: output_columns, + } } } From bc4efde38b6aed20a9921ffde0bcd8c24616ea8a Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Sun, 13 Jul 2025 09:38:52 +0800 Subject: [PATCH 34/37] add log --- .../src/schedulers/fragments/query_fragment_actions.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs index ea2dccf3d2379..010067c701ce1 100644 --- a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs +++ b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs @@ -130,6 +130,13 @@ impl QueryFragmentsActions { pub fn get_root_fragment_ids(&self) -> Result> { let mut fragment_ids = Vec::new(); for fragment_actions in &self.fragments_actions { + if fragment_actions.fragment_actions.is_empty() { + return Err(ErrorCode::Internal(format!( + "Fragment actions is empty for fragment_id: {}", + fragment_actions.fragment_id + ))); + } + let plan = &fragment_actions.fragment_actions[0].physical_plan; if !matches!(plan, PhysicalPlan::ExchangeSink(_)) { fragment_ids.push(fragment_actions.fragment_id); From b5ccca9332c0bef7a9aa1a2091afac7c893377c8 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Sun, 13 Jul 2025 15:01:59 +0800 Subject: [PATCH 35/37] fix --- .../src/schedulers/fragments/fragmenter.rs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index 2e7a6e878b819..eaef119265cf4 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -26,6 +26,7 @@ use databend_common_sql::executor::physical_plans::ExchangeSink; use databend_common_sql::executor::physical_plans::ExchangeSource; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::HashJoin; +use databend_common_sql::executor::physical_plans::MaterializedCTE; use databend_common_sql::executor::physical_plans::MutationSource; use databend_common_sql::executor::physical_plans::Recluster; use databend_common_sql::executor::physical_plans::ReplaceInto; @@ -341,4 +342,23 @@ impl PhysicalPlanReplacer for Fragmenter { source_fragment_id, })) } + + fn replace_materialized_cte(&mut self, plan: &MaterializedCTE) -> Result { + let mut fragments = vec![]; + let left = self.replace(plan.left.as_ref())?; + + fragments.append(&mut self.fragments); + let right = self.replace(plan.right.as_ref())?; + fragments.append(&mut self.fragments); + + self.fragments = fragments; + Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { + plan_id: plan.plan_id, + left: Box::new(left), + right: Box::new(right), + stat_info: plan.stat_info.clone(), + cte_name: plan.cte_name.clone(), + cte_output_columns: plan.cte_output_columns.clone(), + }))) + } } From 2f897f4e06556a410ff41a7e98c285d7338b01a3 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Sun, 13 Jul 2025 16:29:04 +0800 Subject: [PATCH 36/37] fix --- src/query/service/src/schedulers/fragments/fragmenter.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index eaef119265cf4..29b36f051b3ce 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_meta_types::NodeInfo; +use databend_common_sql::executor::physical_plans::CTEConsumer; use databend_common_sql::executor::physical_plans::CompactSource; use databend_common_sql::executor::physical_plans::ConstantTableScan; use databend_common_sql::executor::physical_plans::CopyIntoTable; @@ -68,6 +69,7 @@ enum State { Compact, Recluster, Other, + CTEConsumer, } impl Fragmenter { @@ -161,6 +163,11 @@ impl PhysicalPlanReplacer for Fragmenter { Ok(PhysicalPlan::TableScan(plan.clone())) } + fn replace_cte_consumer(&mut self, plan: &CTEConsumer) -> Result { + self.state = State::CTEConsumer; + Ok(PhysicalPlan::CTEConsumer(Box::new(plan.clone()))) + } + fn replace_constant_table_scan(&mut self, plan: &ConstantTableScan) -> Result { self.state = State::SelectLeaf; Ok(PhysicalPlan::ConstantTableScan(plan.clone())) @@ -311,6 +318,7 @@ impl PhysicalPlanReplacer for Fragmenter { State::ReplaceInto => FragmentType::ReplaceInto, State::Compact => FragmentType::Compact, State::Recluster => FragmentType::Recluster, + State::CTEConsumer => FragmentType::Intermediate, }; self.state = State::Other; let exchange = Self::get_exchange(self.ctx.clone(), &plan)?; From cf7e7f97cc4a1960f19519de17e68c776465fa83 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Sun, 13 Jul 2025 16:30:30 +0800 Subject: [PATCH 37/37] make lint --- .../service/src/schedulers/fragments/query_fragment_actions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs index 010067c701ce1..9c0d5d42f888e 100644 --- a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs +++ b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs @@ -136,7 +136,7 @@ impl QueryFragmentsActions { fragment_actions.fragment_id ))); } - + let plan = &fragment_actions.fragment_actions[0].physical_plan; if !matches!(plan, PhysicalPlan::ExchangeSink(_)) { fragment_ids.push(fragment_actions.fragment_id);